Merge pull request #36 from gao-feng/scsi

find scsi disk device if scsi addr is given

.gitignore

@@ -10,6 +10,7 @@
 init
 build/hyper_daemon
 build/hyper-initrd.img
+build/cbfs.rom
 build/root/*
 build/*iso
 build/daemon

README.md

@@ -1,21 +1,21 @@
 # The init Task for Hyper
 
-You can get binary installer of Hyper and HyperStart through [The Hyper Page](https://github.com/hyperhq/hyper)
+You can get the binary installer of Hyper and HyperStart through [The Hyper Page](https://github.com/hyperhq/hyper)
 
 ## Build from source 
 
-clone this repo, and make sure have build-essentials installed. Go into the working copy and
+clone this repo, and make sure you have build-essentials installed. Go into the working copy and
 
     > ./autogen.sh
     > ./configure
     > make
 
-Then you can find `hyper-initrd.img` in build directory, together with a pre-build kernel.
+Then you can find `hyper-initrd.img` in the build directory, together with a pre-built kernel.
 
-If you want to get the boot disk file for VirtaulBox, please reconfigure with flag --with-vbox,
+If you want to get the boot disk file for VirtualBox, please reconfigure with flag --with-vbox,
 
     > ./configure --with-vbox
     > make
 
-Then you can find `hyper-vbox-bootimage.iso` in build directory, booting from this iso will
+Then you can find `hyper-vbox-bootimage.iso` in the build directory. Booting from this iso will
 bring you to the hyper world.

build/make-initrd.sh

@@ -39,6 +39,8 @@ if [ "$1"x = "cbfs"x ]; then
 	cbfstool .cbfs/cbfs.rom create -s 4096k -B .cbfs/boot.bin -m x86  0x1000
 	cbfstool .cbfs/cbfs.rom add -f kernel -n vmlinuz -t raw
 	cbfstool .cbfs/cbfs.rom add -f hyper-initrd.img -n initrd -t raw
+	echo 'console=ttyS0 panic=1 no_timer_check' > .cbfs/cmdline
+	cbfstool .cbfs/cbfs.rom add -f .cbfs/cmdline -n cmdline -t raw
 	cp .cbfs/cbfs.rom ./
 	rm -rf .cbfs
 	exit 0

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([hyperstart], [0.2], [www.hyper.sh])
+AC_INIT([hyperstart], [0.3], [www.hyper.sh])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects])
 AM_EXTRA_RECURSIVE_TARGETS([initrd cbfs])
 AC_CONFIG_SRCDIR([src/init.c])
@@ -32,6 +32,7 @@ AC_TYPE_UINT8_T
 # Checks for library functions.
 AC_FUNC_FORK
 AC_CHECK_FUNCS([dup2 memmove memset mkdir setenv socket strchr strdup strrchr strtoul], [fail=0], [fail=1])
+AC_CHECK_FUNCS([setns])
 
 if test "$fail" = "1" ; then
     AC_MSG_ERROR(Unable to find necessary functions)

src/container.c

@@ -16,20 +16,8 @@
 
 #include "util.h"
 #include "hyper.h"
-
+#include "parse.h"
-static int container_setup_env(struct hyper_container *container)
+#include "syscall.h"
-{
-	int i;
-	struct env *env;
-
-	for (i = 0; i < container->envs_num; i++) {
-		env = &container->envs[i];
-
-		setenv(env->env, env->value, 1);
-	}
-
-	return 0;
-}
 
 static int container_setup_volume(struct hyper_container *container)
 {
@@ -40,6 +28,9 @@ static int container_setup_volume(struct hyper_container *container)
 	for (i = 0; i < container->vols_num; i++) {
 		vol = &container->vols[i];
 
+		if (vol->scsiaddr)
+			hyper_find_sd("/.oldroot", vol->scsiaddr, &vol->device);
+
 		sprintf(dev, "/.oldroot/dev/%s", vol->device);
 		sprintf(path, "/tmp/%s", vol->mountpoint);
 		fprintf(stdout, "mount %s to %s, tmp path %s\n",
@@ -116,13 +107,23 @@ static int container_setup_mount(struct hyper_container *container)
 	hyper_mkdir("/sys");
 	hyper_mkdir("/dev");
 
-	if (mount("proc", "/proc", "proc", 0, NULL) < 0 ||
+	if (mount("proc", "/proc", "proc", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) < 0 ||
-	    mount("sysfs", "/sys", "sysfs", 0, NULL) < 0 ||
+	    mount("sysfs", "/sys", "sysfs", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) < 0 ||
-	    mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) < 0) {
+	    mount("devtmpfs", "/dev", "devtmpfs", MS_NOSUID, NULL) < 0) {
 		perror("mount basic filesystem for container failed");
 		return -1;
 	}
 
+	if (hyper_mkdir("/dev/shm") < 0) {
+		fprintf(stderr, "create /dev/shm failed\n");
+		return -1;
+	}
+
+	if (mount("tmpfs", "/dev/shm/", "tmpfs", MS_NOSUID| MS_NODEV, NULL) < 0) {
+		perror("mount shm failed");
+		return -1;
+	}
+
 	if (hyper_mkdir("/dev/pts") < 0) {
 		fprintf(stderr, "create /dev/pts failed\n");
 		return -1;
@@ -181,6 +182,78 @@ static int container_setup_mount(struct hyper_container *container)
 	return 0;
 }
 
+static int container_setup_sysctl(struct hyper_container *container)
+{
+	int i, size, len, l, fd;
+	struct sysctl *sys;
+
+	for (i = 0; i < container->sys_num; i++) {
+		char path[256];
+
+		len = 0;
+		sys = &container->sys[i];
+		size = strlen(sys->value);
+
+		sprintf(path, "/proc/sys/%s", sys->path);
+		fprintf(stdout, "sysctl %s value %s\n", sys->path, sys->value);
+
+		fd = open(path, O_WRONLY);
+		if (fd < 0) {
+			perror("open file failed");
+			goto out;
+		}
+
+		while (len < size) {
+			l = write(fd, sys->value + len, size - len);
+			if (l < 0) {
+				perror("fail to write sysctl");
+				close(fd);
+				goto out;
+			}
+			len += l;
+		}
+
+		close(fd);
+	}
+
+	return 0;
+out:
+	return -1;
+}
+
+static int container_setup_dns(struct hyper_container *container)
+{
+	int fd;
+	struct stat st;
+	char *src = "/.oldroot/tmp/hyper/resolv.conf";
+
+	if (stat(src, &st) < 0) {
+		if (errno == ENOENT) {
+			fprintf(stdout, "no dns configured\n");
+			return 0;
+		}
+
+		perror("stat resolve.conf failed");
+		return -1;
+	}
+
+	hyper_mkdir("/etc");
+
+	fd = open("/etc/resolv.conf", O_CREAT| O_WRONLY, 0644);
+	if (fd < 0) {
+		perror("create /etc/resolv.conf failed");
+		return -1;
+	}
+	close(fd);
+
+	if (mount(src, "/etc/resolv.conf", NULL, MS_BIND, NULL) < 0) {
+		perror("bind to /etc/resolv.conf failed");
+		return -1;
+	}
+
+	return 0;
+}
+
 static int container_setup_workdir(struct hyper_container *container)
 {
 	if (container->workdir && chdir(container->workdir) < 0) {
@@ -243,6 +316,8 @@ static int hyper_rescan_scsi(void)
 
 struct hyper_container_arg {
 	struct hyper_container	*c;
+	int			ipcns;
+	int			utsns;
 	int			pipe[2];
 };
 
@@ -258,12 +333,22 @@ static int hyper_container_init(void *data)
 		goto fail;
 	}
 
+	if (setns(arg->ipcns, CLONE_NEWIPC) < 0) {
+		perror("setns to ipcns of pod init faild");
+		goto fail;
+	}
+
+	if (setns(arg->utsns, CLONE_NEWUTS) < 0) {
+		perror("setns to ipcns of pod init faild");
+		goto fail;
+	}
+
 	if (hyper_rescan_scsi() < 0) {
 		fprintf(stdout, "rescan scsi failed\n");
 		goto fail;
 	}
 
-	if (container_setup_env(container) < 0) {
+	if (hyper_setup_env(container->envs, container->envs_num) < 0) {
 		fprintf(stdout, "setup env failed\n");
 		goto fail;
 	}
@@ -289,6 +374,9 @@ static int hyper_container_init(void *data)
 	if (container->fstype) {
 		char dev[128];
 
+		if (container->scsiaddr)
+			hyper_find_sd("", container->scsiaddr, &container->image);
+
 		sprintf(dev, "/dev/%s", container->image);
 		fprintf(stdout, "device %s\n", dev);
 
@@ -335,7 +423,17 @@ static int hyper_container_init(void *data)
 	}
 
 	if (container_setup_mount(container) < 0) {
-		fprintf(stderr, "container sets up mount ns failed\n");
+		fprintf(stderr, "container sets up mount failed\n");
+		goto fail;
+	}
+
+	if (container_setup_sysctl(container) < 0) {
+		fprintf(stderr, "container sets up sysctl failed\n");
+		goto fail;
+	}
+
+	if (container_setup_dns(container) < 0) {
+		fprintf(stderr, "container sets up dns failed\n");
 		goto fail;
 	}
 
@@ -353,8 +451,10 @@ static int hyper_container_init(void *data)
 		goto fail;
 	}
 
-	close(arg->pipe[0]);
+	symlink("/proc/self/fd", "/dev/fd");
-	close(arg->pipe[1]);
+	symlink("/proc/self/fd/0", "/dev/stdin");
+	symlink("/proc/self/fd/1", "/dev/stdout");
+	symlink("/proc/self/fd/2", "/dev/stderr");
 
 	execvp(container->exec.argv[0], container->exec.argv);
 	perror("exec container command failed");
@@ -362,20 +462,49 @@ static int hyper_container_init(void *data)
 	_exit(-1);
 
 fail:
-	container->exec.code = -1;
+	hyper_send_type(arg->pipe[1], ERROR);
-	hyper_send_type_block(arg->pipe[1], ERROR, 0);
 	_exit(-1);
 }
 
-int hyper_start_container(struct hyper_container *container)
+static int hyper_setup_pty(struct hyper_container *c)
+{
+	char root[512];
+
+	sprintf(root, "/tmp/hyper/%s/devpts/", c->id);
+
+	if (hyper_mkdir(root) < 0) {
+		perror("make container pts directroy failed");
+		return -1;
+	}
+
+	if (mount("devpts", root, "devpts", MS_NOSUID,
+		  "newinstance,ptmxmode=0666,mode=0620") < 0) {
+		perror("mount devpts failed");
+		return -1;
+	}
+
+	if (hyper_setup_exec_tty(&c->exec) < 0) {
+		fprintf(stderr, "setup container pts failed\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int hyper_start_container(struct hyper_container *container,
+			  int utsns, int ipcns, struct hyper_pod *pod)
 {
 	int stacksize = getpagesize() * 4;
-	void *stack = malloc(stacksize);
 	struct hyper_container_arg arg = {
-		.c = container,
+		.c	= container,
+		.utsns	= utsns,
+		.ipcns	= ipcns,
+		.pipe	= {-1, -1},
 	};
 	int flags = CLONE_NEWNS | SIGCHLD;
+	char path[128];
 	uint32_t type;
+	void *stack;
 	int pid;
 
 	if (container->image == NULL || container->exec.argv == NULL) {
@@ -384,151 +513,105 @@ int hyper_start_container(struct hyper_container *container)
 		goto fail;
 	}
 
-	if (socketpair(PF_UNIX, SOCK_STREAM, 0, arg.pipe) < 0) {
+	if (hyper_setup_pty(container) < 0) {
+		fprintf(stderr, "setup pty device for container failed\n");
+		goto fail;
+	}
+
+	if (pipe2(arg.pipe, O_CLOEXEC) < 0) {
 		perror("create pipe between pod init execcmd failed");
 		goto fail;
 	}
 
+	if (hyper_watch_exec_pty(&container->exec, pod) < 0) {
+		fprintf(stderr, "faile to watch container pty\n");
+		goto fail;
+	}
+
+	stack = malloc(stacksize);
+	if (stack == NULL) {
+		perror("fail to allocate stack for container init");
+		goto fail;
+	}
+
 	pid = clone(hyper_container_init, stack + stacksize, flags, &arg);
 	free(stack);
 	if (pid < 0) {
 		perror("create child process failed");
 		goto fail;
 	}
+	sprintf(path, "/proc/%d/ns/mnt", pid);
 
-	container->exec.pid = pid;
+	container->ns = open(path, O_RDONLY | O_CLOEXEC);
-
+	if (container->ns < 0) {
-	/* wait for ready message */
+		perror("open container mount ns failed");
-	if (hyper_get_type_block(arg.pipe[0], &type) < 0 || type != READY) {
-		fprintf(stdout, "wait for container started failed\n");
 		goto fail;
 	}
 
+	/* wait for ready message */
+	if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) {
+		fprintf(stderr, "wait for container started failed\n");
+		goto fail;
+	}
+
+	container->exec.pid = pid;
+	list_add_tail(&container->exec.list, &pod->exec_head);
+	container->exec.ref++;
+
 	close(arg.pipe[0]);
 	close(arg.pipe[1]);
 
-	fprintf(stdout, "container %s init pid is %d\n", container->id, pid);
+	fprintf(stdout, "container %s,init pid %d,ref %d\n", container->id, pid, container->exec.ref);
 	return 0;
-
 fail:
+	close(arg.pipe[0]);
+	close(arg.pipe[1]);
+	close(container->ns);
+	hyper_reset_event(&container->exec.e);
+	hyper_reset_event(&container->exec.errev);
+	container->ns = -1;
 	fprintf(stdout, "container %s init exit code %d\n", container->id, -1);
 	container->exec.code = -1;
+	container->exec.seq = 0;
+	container->exec.ref = 0;
 	return -1;
 }
 
-int hyper_start_containers(struct hyper_pod *pod)
-{
-	int i;
-
-	/* mount new proc directory */
-	if (umount("/proc") < 0) {
-		perror("umount proc filesystem failed\n");
-		return -1;
-	}
-
-	if (mount("proc", "/proc", "proc", 0, NULL) < 0) {
-		perror("mount proc filesystem failed\n");
-		return -1;
-	}
-
-	if (sethostname(pod->hostname, strlen(pod->hostname)) < 0) {
-		perror("set host name failed");
-		return -1;
-	}
-
-	for (i = 0; i < pod->c_num; i++)
-		hyper_start_container(&pod->c[i]);
-
-	return 0;
-}
-
-int hyper_restart_containers(struct hyper_pod *pod)
-{
-	int i;
-	struct hyper_container *c;
-
-	for (i = 0; i < pod->c_num; i++) {
-		c = &pod->c[i];
-
-		if (hyper_start_container(c) < 0) {
-			fprintf(stderr, "restart container %s failed\n", c->id);
-			hyper_send_type(pod->ctl.fd, ERROR);
-			return -1;
-		}
-	}
-
-	if (hyper_send_type(pod->ctl.fd, ACK) < 0)
-		return -1;
-
-	return 0;
-}
-
 struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id)
 {
-	int i;
+	struct hyper_container *c;
-	struct hyper_container *container;
 
-	for (i = 0; i < pod->c_num; i++) {
+	list_for_each_entry(c, &pod->containers, list) {
-		container = &pod->c[i];
+		if (strlen(c->id) != strlen(id))
-
-		if (strlen(container->id) != strlen(id))
 			continue;
 
-		if (strncmp(container->id, id, strlen(id)))
+		if (strncmp(c->id, id, strlen(id)))
 			continue;
 
-		return container;
+		return c;
 	}
 
 	return NULL;
 }
 
-void hyper_cleanup_container(struct hyper_pod *pod)
+void hyper_cleanup_container(struct hyper_container *c)
 {
-	int i, j;
-	struct hyper_container *c;
-	struct volume *vol;
-	struct env *env;
-	struct fsmap *map;
 	char root[512];
 
-	for (i = 0; i < pod->c_num; i++) {
+	sprintf(root, "/tmp/hyper/%s/devpts/", c->id);
-		c = &pod->c[i];
+	if (umount(root) < 0 && umount2(root, MNT_DETACH))
+		perror("umount devpts failed");
 
-		sprintf(root, "/tmp/hyper/%s/devpts/", c->id);
+	close(c->ns);
-		if (umount(root) < 0 && umount2(root, MNT_DETACH))
+	hyper_free_container(c);
-			perror("umount devpts failed");
+}
 
-		free(c->id);
+void hyper_cleanup_containers(struct hyper_pod *pod)
-		free(c->rootfs);
+{
-		free(c->image);
+	struct hyper_container *c, *n;
-		free(c->workdir);
-		free(c->fstype);
 
-		for (j = 0; j < c->vols_num; j++) {
+	list_for_each_entry_safe(c, n, &pod->containers, list)
-			vol = &(c->vols[j]);
+		hyper_cleanup_container(c);
-			free(vol->device);
-			free(vol->mountpoint);
-			free(vol->fstype);
-		}
-		free(c->vols);
 
-		for (j = 0; j < c->envs_num; j++) {
+	pod->remains = 0;
-			env = &(c->envs[j]);
-			free(env->env);
-			free(env->value);
-		}
-		free(c->envs);
-
-		for (j = 0; j < c->maps_num; j++) {
-			map = &(c->maps[j]);
-			free(map->source);
-			free(map->path);
-		}
-		free(c->maps);
-	}
-
-	free(pod->c);
-	pod->c = NULL;
-	pod->c_num = 0;
 }

src/container.h

@@ -10,6 +10,7 @@ struct env {
 
 struct volume {
 	char	*device;
+	char	*scsiaddr;
 	char	*mountpoint;
 	char	*fstype;
 	int	readonly;
@@ -21,27 +22,39 @@ struct fsmap {
 	int	readonly;
 };
 
+struct sysctl {
+	char	*path;
+	char	*value;
+};
+
 struct hyper_container {
 	char			*id;
 	char			*rootfs;
 	char			*image;
+	char			*scsiaddr;
 	char			*workdir;
 	char			*fstype;
 	struct volume		*vols;
 	struct env		*envs;
 	struct fsmap		*maps;
+	struct sysctl		*sys;
 	int			vols_num;
 	int			envs_num;
 	int			maps_num;
+	int			sys_num;
+	int			ns;
 	uint32_t		code;
+	struct list_head	list;
 	struct hyper_exec	exec;
 };
 
 struct hyper_pod;
 
-int hyper_start_containers(struct hyper_pod *pod);
+int hyper_start_container(struct hyper_container *container,
+			  int utsns, int ipcns, struct hyper_pod *pod);
 struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id);
-int hyper_restart_containers(struct hyper_pod *pod);
+void hyper_cleanup_container(struct hyper_container *container);
-void hyper_cleanup_container(struct hyper_pod *pod);
+void hyper_cleanup_containers(struct hyper_pod *pod);
+void hyper_free_container(struct hyper_container *c);
 
 #endif

src/event.c

@@ -14,8 +14,9 @@ void hyper_reset_event(struct hyper_event *de)
 {
 	free(de->rbuf.data);
 	free(de->wbuf.data);
-
+	close(de->fd);
 	memset(de, 0, sizeof(*de));
+	de->fd = -1;
 }
 
 int hyper_init_event(struct hyper_event *de, struct hyper_event_ops *ops, void *arg)
@@ -219,36 +220,40 @@ void hyper_event_hup(struct hyper_event *de, int efd)
 {
 	if (epoll_ctl(efd, EPOLL_CTL_DEL, de->fd, NULL) < 0)
 		perror("epoll_ctl del epoll event failed");
-	close(de->fd);
 	hyper_reset_event(de);
 }
 
 int hyper_handle_event(int efd, struct epoll_event *event)
 {
 	struct hyper_event *de = event->data.ptr;
+	fprintf(stdout, "%s get event %d, de %p, fd %d. ops %p\n",
+			__func__, event->events, de, de->fd, de->ops);
 
-	if (event->events & EPOLLHUP) {
+	/* do not handle hup event if have in event */
+	if (event->events & EPOLLIN) {
+		fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n",
+			__func__, de, de->fd, de->ops);
+		if (de->ops->read(de) < 0)
+			return -1;
+	} else if (event->events & EPOLLHUP) {
 		fprintf(stdout, "%s event EPOLLHUP, de %p, fd %d, %p\n",
 			__func__, de, de->fd, de->ops);
 		if (de->ops->hup)
 			de->ops->hup(de, efd);
 		return 0;
-	} else if (event->events & EPOLLIN) {
+	}
-		fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n",
+
-			__func__, de, de->fd, de->ops);
+	if (event->events & EPOLLOUT) {
-		return de->ops->read(de);
-	} else if (event->events & EPOLLOUT) {
 		fprintf(stdout, "%s event EPOLLOUT, de %p, fd %d, %p\n",
 			__func__, de, de->fd, de->ops);
-		if (de->ops->write)
+		if (de->ops->write && de->ops->write(de) < 0)
-			return de->ops->write(de);
+			return -1;
-		fprintf(stderr, "warning: %p received unexpected write event\n", de);
+	}
-		return 0;
+
-	} else if (event->events & EPOLLERR) {
+	if (event->events & EPOLLERR) {
 		fprintf(stderr, "get epoll err of not epool in event\n");
 		return -1;
 	}
 
-	fprintf(stdout, "%s get unknown event %d\n", __func__, event->events);
+	return 0;
-	return -1;
 }

src/exec.c

@@ -9,41 +9,84 @@
 #include <sched.h>
 #include <errno.h>
 #include <string.h>
+#include <signal.h>
 #include <fcntl.h>
 #include <inttypes.h>
-#include "syscall.h"
 
 #include "hyper.h"
 #include "util.h"
 #include "parse.h"
+#include "syscall.h"
 
-static int pts_loop(struct hyper_event *de)
+static void pts_hup(struct hyper_event *de, int efd, int out)
 {
-	int size = 0, i;
+	struct hyper_exec *exec;
+	struct hyper_pod *pod = de->ptr;
 	struct hyper_buf *buf = &ctl.tty.wbuf;
-	struct hyper_exec *exec = container_of(de, struct hyper_exec, e);
+	uint64_t seq;
 
-	dprintf("%s\n", __func__);
+	if (out) {
-	while (buf->get + 12 < buf->size) {
+		exec = container_of(de, struct hyper_exec, e);
+		seq = exec->seq;
+	} else {
+		exec = container_of(de, struct hyper_exec, errev);
+		seq = exec->errseq;
+	}
+
+	fprintf(stdout, "%s, seq %" PRIu64"\n", __func__, seq);
+
+	hyper_event_hup(de, efd);
+
+	if (buf->get + 12 > buf->size) {
+		fprintf(stdout, "%s: tty buf full\n", __func__);
+		return;
+	}
+
+	/* no in event, no more data, send eof */
+	hyper_set_be64(buf->data + buf->get, seq);
+	hyper_set_be32(buf->data + buf->get + 8, 12);
+	buf->get += 12;
+
+	hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT);
+
+	hyper_release_exec(exec, pod);
+}
+
+static void stdout_hup(struct hyper_event *de, int efd)
+{
+	fprintf(stdout, "%s\n", __func__);
+	return pts_hup(de, efd, 1);
+}
+
+static void stderr_hup(struct hyper_event *de, int efd)
+{
+	fprintf(stdout, "%s\n", __func__);
+	return pts_hup(de, efd, 0);
+}
+
+static int pts_loop(struct hyper_event *de, uint64_t seq)
+{
+	int size = -1;
+	struct hyper_buf *buf = &ctl.tty.wbuf;
+
+	while ((buf->get + 12 < buf->size) && size) {
 		size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12);
-		dprintf("%s: read %d data\n", __func__, size);
+		fprintf(stdout, "%s: read %d data\n", __func__, size);
 		if (size <= 0) {
 			if (errno == EINTR)
 				continue;
-			if (errno == EAGAIN || errno == EIO)
-				break;
 
-			perror("fail to read tty fd");
+			if (errno != EAGAIN && errno != EIO) {
-			return -1;
+				perror("fail to read tty fd");
+				return -1;
+			}
+
+			break;
 		}
 
-		hyper_set_be64(buf->data + buf->get, exec->seq);
+		hyper_set_be64(buf->data + buf->get, seq);
 		hyper_set_be32(buf->data + buf->get + 8, size + 12);
 		buf->get += size + 12;
-
-		dprintf("%s: seq %" PRIu64" len %" PRIu32"\n", __func__, exec->seq, size);
-		for (i = 0; i < size; i++)
-			dprintf("%0x ", buf->data[i]);
 	}
 
 	if (hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT) < 0) {
@@ -54,14 +97,37 @@ static int pts_loop(struct hyper_event *de)
 	return 0;
 }
 
+static int stdout_loop(struct hyper_event *de)
+{
+	struct hyper_exec *exec = container_of(de, struct hyper_exec, e);
+	fprintf(stdout, "%s, seq %" PRIu64"\n", __func__, exec->seq);
+
+	return pts_loop(de, exec->seq);
+}
+
 struct hyper_event_ops pts_ops = {
-	.read		= pts_loop,
+	.read		= stdout_loop,
+	.hup		= stdout_hup,
 	.write		= hyper_event_write,
-	.hup		= hyper_event_hup,
 	.wbuf_size	= 512,
 	/* don't need read buff, the pts data will store in tty buffer */
 };
 
+static int stderr_loop(struct hyper_event *de)
+{
+	struct hyper_exec *exec = container_of(de, struct hyper_exec, errev);
+	fprintf(stdout, "%s, seq %" PRIu64"\n", __func__, exec->errseq);
+
+	return pts_loop(de, exec->errseq);
+}
+
+struct hyper_event_ops err_ops = {
+	.read		= stderr_loop,
+	.hup		= stderr_hup,
+	/* don't need read buff, the stderr data will store in tty buffer */
+	/* don't need write buff, the stderr data is one way */
+};
+
 int hyper_setup_exec_tty(struct hyper_exec *e)
 {
 	int unlock = 0;
@@ -70,6 +136,16 @@ int hyper_setup_exec_tty(struct hyper_exec *e)
 	if (e->seq == 0)
 		return 0;
 
+	if (e->errseq > 0) {
+		int errpipe[2];
+		if (pipe2(errpipe, O_NONBLOCK|O_CLOEXEC) < 0) {
+			fprintf(stderr, "creating stderr pipe failed\n");
+			return -1;
+		}
+		e->errev.fd = errpipe[0];
+		e->errfd = errpipe[1];
+	}
+
 	if (e->id) {
 		if (sprintf(path, "/tmp/hyper/%s/devpts/", e->id) < 0) {
 			fprintf(stderr, "get ptmx path failed\n");
@@ -108,335 +184,382 @@ int hyper_setup_exec_tty(struct hyper_exec *e)
 		return -1;
 	}
 
-	e->pty = strdup(ptmx);
+	e->ptyfd = open(ptmx, O_RDWR | O_NOCTTY | O_CLOEXEC);
-	fprintf(stdout, "get pty device for exec %s\n", e->pty);
+	fprintf(stdout, "get pty device for exec %s\n", ptmx);
 
+	fprintf(stdout, "%s pts event %p, fd %d %d\n",
+		__func__, &e->e, e->e.fd, e->ptyfd);
 	return 0;
 }
 
 int hyper_dup_exec_tty(int to, struct hyper_exec *e)
 {
-	int fd;
+	int fd = -1, ret = -1;
 	char pty[128];
 
+	fprintf(stdout, "%s\n", __func__);
 	setsid();
 
 	if (e->seq) {
-		if (sprintf(pty, "/dev/pts/%d", e->ptyno) < 0) {
+		fd = e->ptyfd;
-			perror("get pts device name failed");
-			return -1;
-		}
 	} else {
 		if (sprintf(pty, "/dev/null") < 0) {
 			perror("get pts device name failed");
-			return -1;
+			goto out;
 		}
+		fd = open(pty, O_RDWR | O_NOCTTY);
 	}
 
-	fprintf(stdout, "setup pty device %s for exec\n", pty);
-
-	fd = open(pty, O_RDWR | O_NOCTTY);
 	if (fd < 0) {
 		perror("open pty device for execcmd failed");
-		return -1;
+		goto out;
 	}
 
 	if (e->seq && (ioctl(fd, TIOCSCTTY, NULL) < 0)) {
 		perror("ioctl pty device for execcmd failed");
-		return -1;
+		goto out;
-	}
-
-	if (hyper_send_type_block(to, READY, 0) < 0) {
-		fprintf(stderr, "send ready message to hyper init failed\n");
-		return -1;
 	}
 
 	fflush(stdout);
+	hyper_send_type(to, READY);
 
 	if (dup2(fd, STDIN_FILENO) < 0) {
 		perror("dup tty device to stdin failed");
-		close(fd);
+		goto out;
-		return -1;
 	}
 
 	if (dup2(fd, STDOUT_FILENO) < 0) {
 		perror("dup tty device to stdout failed");
-		close(fd);
+		goto out;
-		return -1;
 	}
 
-	if (dup2(fd, STDERR_FILENO) < 0) {
+	if (e->errseq > 0) {
-		perror("dup tty device to stderr failed");
+		if (dup2(e->errfd, STDERR_FILENO) < 0) {
-		close(fd);
+			perror("dup err pipe to stderr failed");
-		return -1;
+			goto out;
+		}
+	} else {
+		if (dup2(fd, STDERR_FILENO) < 0) {
+			perror("dup tty device to stderr failed");
+			goto out;
+		}
 	}
 
+	ret = 0;
+out:
 	close(fd);
 
-	return 0;
+	return ret;
 }
 
-int hyper_exec_in_container(struct hyper_pod *pod,
+int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod)
-			    struct hyper_exec *exec)
 {
-	global_exec = exec;
+	fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n",
+		&exec->e, &pts_ops, exec->e.fd);
 
-	if (hyper_send_type_block(ctl.ctl.fd, EXECCMD, 1) < 0) {
-		global_exec = NULL;
-		fprintf(stderr, "tell pod init EXECCMD failed\n");
-		return -1;
-	}
-
-	list_add_tail(&exec->list, &pod->ce_head);
 	if (exec->seq == 0)
 		return 0;
 
-	fprintf(stdout, "init container exec pts event %p, ops %p, fd %d\n",
-		&exec->e, &pts_ops, exec->e.fd);
-
 	if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 ||
 	    hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) {
-		fprintf(stderr, "add pts master event failed\n");
+		fprintf(stderr, "add container pts master event failed\n");
 		return -1;
 	}
+	exec->ref++;
 
+	if (exec->errseq == 0)
+		return 0;
+
+	if (hyper_init_event(&exec->errev, &err_ops, pod) < 0 ||
+	    hyper_add_event(ctl.efd, &exec->errev, EPOLLIN) < 0) {
+		fprintf(stderr, "add container stderr event failed\n");
+		return -1;
+	}
+	exec->ref++;
 	return 0;
 }
 
-int hyper_request_restart_containers(struct hyper_pod *pod)
+int hyper_enter_container(struct hyper_pod *pod,
+			  struct hyper_exec *exec)
 {
-	int i;
+	int ipcns, utsns, mntns, ret;
-	struct hyper_exec *exec;
+	struct hyper_container *c;
+	char path[512];
 
-	pod->code = 0;
+	ret = ipcns = utsns = mntns = -1;
-	pod->remains = pod->c_num;
 
-	for (i = 0; i < pod->c_num; i++) {
+	c = hyper_find_container(pod, exec->id);
-		exec = &pod->c[i].exec;
+	if (c == NULL) {
-
+		fprintf(stderr, "can not find container %s\n", exec->id);
-		if (hyper_setup_exec_tty(exec) < 0) {
-			fprintf(stdout, "restart setup container tty failed\n");
-			return -1;
-		}
-	}
-
-	if (hyper_send_type_block(ctl.ctl.fd, RESTARTCONTAINER, 1) < 0) {
-		fprintf(stderr, "tell container init RESTARTCONTAINER failed\n");
 		return -1;
 	}
 
-	for (i = 0; i < pod->c_num; i++) {
+	sprintf(path, "/proc/%d/ns/uts", pod->init_pid);
-		exec = &pod->c[i].exec;
+	utsns = open(path, O_RDONLY| O_CLOEXEC);
-
+	if (utsns < 0) {
-		list_add_tail(&exec->list, &pod->ce_head);
+		perror("fail to open utsns of pod init");
-		if (exec->seq == 0)
+		goto out;
-			continue;
-
-		if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 ||
-		    hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) {
-			fprintf(stderr, "add pts master event failed\n");
-			return -1;
-		}
 	}
 
-	return 0;
+	sprintf(path, "/proc/%d/ns/ipc", pod->init_pid);
+	ipcns = open(path, O_RDONLY| O_CLOEXEC);
+	if (ipcns < 0) {
+		perror("fail to open ipcns of pod init");
+		goto out;
+	}
+
+	mntns = c->ns;
+	if (mntns < 0) {
+		perror("fail to open mntns of pod init");
+		goto out;
+	}
+
+	if (setns(utsns, CLONE_NEWUTS) < 0 ||
+	    setns(ipcns, CLONE_NEWIPC) <0 ||
+	    setns(mntns, CLONE_NEWNS) < 0) {
+		perror("fail to enter container ns");
+		goto out;
+	}
+
+	sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs);
+	fprintf(stdout, "root directory for container is %s, exec %s\n",
+		path, exec->argv[0]);
+
+	/* TODO: wait for container finishing setup root */
+	if (chroot(path) < 0) {
+		perror("chroot for exec command failed");
+		goto out;
+	}
+
+	chdir("/");
+
+	ret = hyper_setup_env(c->envs, c->envs_num);
+out:
+	close(ipcns);
+	close(utsns);
+
+	return ret;
+}
+
+struct hyper_exec_arg {
+	struct hyper_pod	*pod;
+	struct hyper_exec	*exec;
+	int			pipe[2];
+};
+
+static int hyper_do_exec_cmd(void *data)
+{
+	struct hyper_exec_arg *arg = data;
+	struct hyper_exec *exec = arg->exec;
+	struct hyper_pod *pod = arg->pod;
+	int pipe[2] = {-1, -1}, pid;
+	int ret = -1;
+
+	if (exec->id) {
+		char path[512];
+		int pidns;
+
+		sprintf(path, "/proc/%d/ns/pid", pod->init_pid);
+		pidns = open(path, O_RDONLY| O_CLOEXEC);
+		if (pidns < 0) {
+			perror("fail to open pidns of pod init");
+			goto out;
+		}
+
+		/* enter pidns of pod init, so the children of this process will run in
+		 * pidns of pod init, see man 2 setns */
+		if (setns(pidns, CLONE_NEWPID) < 0) {
+			perror("enter pidns of pod init failed");
+			goto out;
+		}
+		close(pidns);
+	}
+
+	if (pipe2(pipe, O_CLOEXEC) < 0) {
+		perror("create pipe in exec command failed");
+		goto out;
+	}
+
+	if (hyper_watch_exec_pty(exec, pod) < 0) {
+		fprintf(stderr, "add pts master event failed\n");
+		goto out;
+	}
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fail to fork");
+		goto out;
+	} else if (pid > 0) {
+		uint32_t type;
+
+		if (hyper_get_type(pipe[0], &type) < 0 || type != READY) {
+			fprintf(stderr, "hyper init doesn't get execcmd ready message\n");
+			hyper_send_type(arg->pipe[1], ERROR);
+			goto out;
+		}
+
+		fprintf(stdout, "hyper init get ready message\n");
+		exec->pid = pid;
+		//TODO combin ref++ and add to list.
+		list_add_tail(&exec->list, &pod->exec_head);
+		exec->ref++;
+		fprintf(stdout, "create exec cmd %s pid %d,ref %d\n", exec->argv[0], pid, exec->ref);
+		ret = 0;
+		goto out;
+	}
+
+	if (exec->id && hyper_enter_container(pod, exec) < 0) {
+		fprintf(stderr, "enter container ns failed\n");
+		goto exit;
+	}
+
+	if (hyper_dup_exec_tty(pipe[1], exec) < 0) {
+		fprintf(stderr, "dup pts to exec stdio failed\n");
+		goto exit;
+	}
+
+	if (execvp(exec->argv[0], exec->argv) < 0) {
+		perror("exec failed");
+		goto exit;
+	}
+
+	ret = 0;
+exit:
+	hyper_send_type(pipe[1], ERROR);
+	_exit(ret);
+
+out:
+	hyper_send_type(arg->pipe[1], ret ? ERROR : READY);
+	close(pipe[0]);
+	close(pipe[1]);
+	_exit(ret);
+}
+
+static void hyper_free_exec(struct hyper_exec *exec)
+{
+	int i;
+
+	free(exec->id);
+
+	for (i = 0; i < exec->argc; i++) {
+		//fprintf(stdout, "argv %d %s\n", i, exec->argv[i]);
+		free(exec->argv[i]);
+	}
+
+	free(exec->argv);
+	free(exec);
 }
 
 int hyper_exec_cmd(char *json, int length)
 {
 	struct hyper_exec *exec;
 	struct hyper_pod *pod = &global_pod;
-	int pid, pipe[2];
+	int stacksize = getpagesize() * 4;
+	void *stack = NULL;
+	struct hyper_exec_arg arg = {
+		.pod	= pod,
+		.exec	= NULL,
+		.pipe	= {-1, -1},
+	};
+	int pid, ret = -1;
+	uint32_t type;
 
 	fprintf(stdout, "call hyper_exec_cmd, json %s, len %d\n", json, length);
 
 	exec = hyper_parse_execcmd(json, length);
 	if (exec == NULL) {
 		fprintf(stderr, "parse exec cmd failed\n");
-		return -1;
+		goto out;
 	}
 
 	if (exec->argv == NULL) {
 		fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n",
 			exec->argv, exec->seq, exec->id);
-		return -1;
+		goto free_exec;
 	}
 
 	if (hyper_setup_exec_tty(exec) < 0) {
 		fprintf(stderr, "setup exec tty failed\n");
-		return -1;
+		goto free_exec;
 	}
 
-	if (exec->id) {
+	if (pipe2(arg.pipe, O_CLOEXEC) < 0) {
-		if (hyper_exec_in_container(pod, exec) < 0) {
-			fprintf(stderr, "notify container exec failed\n");
-			return -1;
-		}
-
-		return 0;
-	}
-
-	if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) {
 		perror("create pipe between pod init execcmd failed");
-		return -1;
+		goto close_tty;
 	}
 
-	pid = fork();
+	arg.exec = exec;
+
+	stack = malloc(stacksize);
+	if (stack == NULL) {
+		perror("fail to allocate stack for container init");
+		goto close_tty;
+	}
+
+	pid = clone(hyper_do_exec_cmd, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg);
+	fprintf(stdout, "do_exec_cmd pid %d\n", pid);
+	free(stack);
 	if (pid < 0) {
-		fprintf(stderr, "fork failed\n");
+		perror("clone hyper_do_exec_cmd failed");
-		return -1;
+		goto close_tty;
-	} else if (pid > 0) {
-		uint32_t type;
-
-		if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) {
-			fprintf(stderr, "hyper init doesn't get execcmd ready message\n");
-			return -1;
-		}
-
-		close(pipe[0]);
-		close(pipe[1]);
-		fprintf(stdout, "hyper init get ready message\n");
-		exec->pid = pid;
-		fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid);
-
-		list_add_tail(&exec->list, &pod->pe_head);
-		if (exec->seq == 0)
-			return 0;
-
-		fprintf(stdout, "init pod exec pts event %p, ops %p, fd %d\n",
-			&exec->e, &pts_ops, exec->e.fd);
-		if (hyper_init_event(&exec->e, &pts_ops, pod) < 0 ||
-		    hyper_add_event(ctl.efd, &exec->e, EPOLLIN) < 0) {
-			fprintf(stderr, "add pts master event failed\n");
-			return -1;
-		}
-
-		return 0;
 	}
 
-	if (hyper_dup_exec_tty(pipe[1], exec) < 0) {
+	if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) {
-		fprintf(stderr, "dup pts to exec stdio failed\n");
+		fprintf(stderr, "hyper init doesn't get execcmd ready message\n");
-		_exit(-1);
+		goto close_tty;
 	}
 
-	close(pipe[0]);
+	fprintf(stdout, "%s get ready message %"PRIu32 "\n", __func__, type);
-	close(pipe[1]);
+	ret = 0;
-
+out:
-	if (execvp(exec->argv[0], exec->argv) < 0) {
+	close(arg.pipe[0]);
-		perror("exec failed");
+	close(arg.pipe[1]);
-		_exit(-1);
+	return ret;
-	}
+close_tty:
-
+	close(exec->ptyfd);
-	_exit(0);
+	close(exec->errfd);
+	close(exec->e.fd);
+free_exec:
+	hyper_free_exec(exec);
+	goto out;
 }
 
-int hyper_container_execcmd(struct hyper_pod *pod)
+static int hyper_send_pod_finished(struct hyper_pod *pod)
 {
-	struct hyper_exec *exec = global_exec;
+	int ret = -1;
-	struct hyper_container *container = NULL;
+	struct hyper_container *c;
-	int fd, pid, sock = pod->ctl.fd;
+	uint8_t *data = NULL, *new;
-	char root[512]; int pipe[2];
+	int c_num = 0;
 
-	global_exec = NULL;
+	list_for_each_entry(c, &pod->containers, list) {
+		c_num++;
+		new = realloc(data, c_num * 4);
+		if (new == NULL)
+			goto out;
 
-	container = hyper_find_container(pod, exec->id);
+		hyper_set_be32(new + ((c_num - 1) * 4), c->exec.code);
-	if (container == NULL) {
+		data = new;
-		fprintf(stderr, "can not find container %s\n", exec->id);
-		return -1;
 	}
 
-	if (sprintf(root, "/tmp/hyper/%s/devpts/", exec->id) < 0) {
+	ret = hyper_send_msg_block(ctl.chan.fd, PODFINISHED, c_num * 4, data);
-		fprintf(stderr, "get container %s pts path failed\n", exec->id);
+out:
-		return -1;
+	free(data);
-	}
+	return ret;
-
-	if (socketpair(PF_UNIX, SOCK_STREAM, 0, pipe) < 0) {
-		perror("create pipe between pod init execcmd failed");
-		return -1;
-	}
-
-	pid = fork();
-	if (pid < 0) {
-		fprintf(stderr, "container init fork failed\n");
-		return -1;
-	} else if (pid > 0) {
-		uint32_t type;
-
-		if (hyper_get_type_block(pipe[0], &type) < 0 || type != READY) {
-			fprintf(stderr, "pod init get execcmd ready message failed\n");
-			hyper_send_type_block(sock, ERROR, 0);
-			return 0;
-		}
-
-		close(pipe[0]);
-		close(pipe[1]);
-		fprintf(stdout, "pod init get ready message\n");
-
-		exec->pid = pid;
-		fprintf(stdout, "create exec cmd %s pid %d\n", exec->argv[0], pid);
-
-		if (hyper_send_type_block(sock, ACK, 0) < 0)
-			return -1;
-
-		return 0;
-	}
-
-	close(pipe[0]);
-	sprintf(root, "/proc/%d/ns/mnt", container->exec.pid);
-
-	fprintf(stdout, "container %s, init pid %d\n",
-		exec->id, container->exec.pid);
-
-	fd = open(root, O_RDONLY);
-	if (fd < 0) {
-		perror("fail to open container mnt ns\n");
-		goto fail;
-	}
-
-	if (syscall(SYS_setns, fd, CLONE_NEWNS) < 0) {
-		perror("enter mnt ns failed");
-		goto fail;
-	}
-
-	close(fd);
-
-	sprintf(root, "/tmp/hyper/%s/root/%s/",
-		container->id, container->rootfs);
-
-	fprintf(stdout, "root directory for container is %s, exec %s\n",
-		root, exec->argv[0]);
-
-	/* TODO: wait for container finishing setup root */
-	if (chroot(root) < 0) {
-		perror("chroot for exec command failed");
-		goto fail;
-	}
-
-	chdir("/");
-
-	if (hyper_dup_exec_tty(pipe[1], exec)) {
-		fprintf(stderr, "dup pts to stdio failed\n");
-		goto fail;
-	}
-
-	close(pipe[1]);
-	if (execvp(exec->argv[0], exec->argv) < 0)
-		perror("exec failed");
-
-	_exit(-1);
-fail:
-	hyper_send_type_block(pipe[1], ERROR, 0);
-	_exit(-1);
 }
 
-
 int hyper_release_exec(struct hyper_exec *exec,
 		       struct hyper_pod *pod)
 {
-	int i;
+	if (--exec->ref != 0) {
+		fprintf(stdout, "still have %d user of exec\n", exec->ref);
+		return 0;
+	}
 
-	close(exec->e.fd);
+	/* exec has no pty or the pty user already exited */
-	free(exec->pty);
+	fprintf(stdout, "last user of exec exit, release\n");
+
+	hyper_reset_event(&exec->e);
+	hyper_reset_event(&exec->errev);
 
 	list_del_init(&exec->list);
 
@@ -445,42 +568,26 @@ int hyper_release_exec(struct hyper_exec *exec,
 		fprintf(stdout, "%s container init exited, type %d, remains %d, policy %d\n",
 			__func__, pod->type, pod->remains, pod->policy);
 
-		/* stop pod, should not restart container */
+		// TODO send finish of this container and full cleanup
-		if (pod->type == STOPPOD)
-			return 0;
-
-		if (exec->code)
-			pod->code = exec->code;
-
 		if (--pod->remains > 0)
 			return 0;
 
-		/* should shutdown? */
+		if (pod->type == STOPPOD) {
-		if (pod->policy == POLICY_NEVER ||
+			/* stop pod manually, hyper doesn't care the pod finished codes */
-		   ((pod->policy == POLICY_ONFAILURE) && pod->code == 0)) {
+			hyper_send_msg_block(ctl.chan.fd, ACK, 0, NULL);
-			hyper_send_finish(pod);
+		} else if (pod->type == DESTROYPOD) {
-			//hyper_shutdown(pod);
+			/* shutdown vm manually, hyper doesn't care the pod finished codes */
-			return 0;
+			hyper_shutdown();
-		}
+		} else {
-
+			/* send out pod finish message, hyper will decide if restart pod or not */
-		if (hyper_request_restart_containers(pod) < 0) {
+			hyper_send_pod_finished(pod);
-			fprintf(stderr, "restart container failed\n");
-			return -1;
 		}
 
+		hyper_cleanup_pod(pod);
 		return 0;
 	}
 
-	free(exec->id);
+	hyper_free_exec(exec);
-
-	for (i = 0; i < exec->argc; i++) {
-		fprintf(stdout, "argv %d %s\n", i, exec->argv[i]);
-		free(exec->argv[i]);
-	}
-
-	free(exec->argv);
-	free(exec);
-
 	return 0;
 }
 
@@ -503,17 +610,8 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq)
 {
 	struct hyper_exec *exec;
 
-	list_for_each_entry(exec, &pod->ce_head, list) {
+	list_for_each_entry(exec, &pod->exec_head, list) {
-		fprintf(stdout, "container exec seq %" PRIu64 ", seq %" PRIu64 "\n",
+		fprintf(stdout, "exec seq %" PRIu64 ", seq %" PRIu64 "\n",
-			exec->seq, seq);
-		if (exec->seq != seq)
-			continue;
-
-		return exec;
-	}
-
-	list_for_each_entry(exec, &pod->pe_head, list) {
-		fprintf(stdout, "pod exec seq %" PRIu64 ", seq %" PRIu64 "\n",
 			exec->seq, seq);
 		if (exec->seq != seq)
 			continue;
@@ -524,35 +622,28 @@ struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq)
 	return NULL;
 }
 
-int hyper_send_exec_eof(int to, struct hyper_pod *pod,
+int hyper_handle_exec_exit(struct hyper_pod *pod, int pid, uint8_t code)
-		      struct list_head *head, int pid,
-		      uint8_t code)
 {
 	struct hyper_exec *exec;
-	uint8_t seq[12];
 
-	exec = hyper_find_exec_by_pid(head, pid);
+	exec = hyper_find_exec_by_pid(&pod->exec_head, pid);
 	if (exec == NULL) {
 		fprintf(stdout, "can not find exec whose pid is %d\n",
 			pid);
 		return 0;
 	}
 
-	fprintf(stdout, "%s exec pid %d, seq %" PRIu64 ", container %s\n",
+	fprintf(stdout, "%s exec exit pid %d, seq %" PRIu64 ", container %s\n",
 		__func__, exec->pid, exec->seq, exec->id ? exec->id : "pod");
 
 	exec->code = code;
+	exec->exit = 1;
 
-	if (exec->seq == 0)
+	close(exec->ptyfd);
-		goto out;
+	exec->ptyfd = -1;
+	close(exec->errfd);
+	exec->errfd = -1;
 
-	hyper_set_be64(seq, exec->seq);
-	hyper_set_be32(seq + 8, 12);
-	if (hyper_send_data(to, seq, 12) < 0) {
-		fprintf(stderr, "pod signal_loop send finishcmd failed\n");
-		return -1;
-	}
-out:
 	hyper_release_exec(exec, pod);
 
 	return 0;
@@ -561,9 +652,16 @@ out:
 void hyper_cleanup_exec(struct hyper_pod *pod)
 {
 	struct hyper_exec *exec, *next;
+	uint8_t buf[12];
 
-	list_for_each_entry_safe(exec, next, &pod->ce_head, list) {
+	if (hyper_setfd_block(ctl.tty.fd) < 0)
-		fprintf(stdout, "cleanup container exec seq %" PRIu64 "\n", exec->seq);
+		return;
-		hyper_release_exec(exec, pod);
+
+	hyper_set_be32(buf + 8, 12);
+	list_for_each_entry_safe(exec, next, &pod->exec_head, list) {
+		fprintf(stdout, "send eof for exec seq %" PRIu64 "\n", exec->seq);
+		hyper_set_be64(buf, exec->seq);
+		if (hyper_send_data(ctl.tty.fd, buf, 12) < 0)
+			fprintf(stderr, "send eof failed\n");
 	}
 }

src/exec.h

@@ -7,15 +7,20 @@
 struct hyper_exec {
 	struct list_head	list;
 	struct hyper_event	e;
+	struct hyper_event	errev;
 	char			*id;
-	char			*pty;
 	char			**argv;
 	int			argc;
 	uint64_t		seq;
+	uint64_t		errseq;
 	int			pid;
 	int			ptyno;
 	int			init;
+	int			ptyfd;
+	int			errfd;
 	uint8_t			code;
+	uint8_t			exit;
+	uint8_t			ref;
 };
 
 struct hyper_pod;
@@ -27,9 +32,8 @@ int hyper_setup_exec_tty(struct hyper_exec *e);
 int hyper_dup_exec_tty(int fd, struct hyper_exec *e);
 struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid);
 struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq);
-int hyper_send_exec_eof(int to, struct hyper_pod *pod,
+int hyper_handle_exec_exit(struct hyper_pod *pod, int pid, uint8_t code);
-		      struct list_head *head, int pid,
+int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod);
-		      uint8_t code);
 void hyper_cleanup_exec(struct hyper_pod *pod);
 
 extern struct hyper_event_ops pts_ops;

src/hyper.h

@@ -1,5 +1,5 @@
-#ifndef _DVM_H_
+#ifndef _HYPER_H_
-#define _DVM_H_
+#define _HYPER_H_
 
 #include <stdint.h>
 
@@ -10,21 +10,25 @@
 #include "container.h"
 
 enum {
-	SETDVM,
+	RESERVED,
 	STARTPOD,
 	GETPOD,
 	STOPPOD,
 	DESTROYPOD,
 	RESTARTCONTAINER,
 	EXECCMD,
-	FINISHCMD,
+	CMDFINISHED,
 	READY,
 	ACK,
 	ERROR,
 	WINSIZE,
 	PING,
-	FINISH,
+	PODFINISHED,
 	NEXT,
+	WRITEFILE,
+	READFILE,
+	NEWCONTAINER,
+	KILLCONTAINER,
 };
 
 enum {
@@ -37,23 +41,22 @@ struct hyper_pod {
 	struct hyper_container	*c;
 	struct hyper_interface	*iface;
 	struct hyper_route	*rt;
-	struct list_head	pe_head;
+	char			**dns;
-	struct list_head	ce_head;
+	struct list_head	containers;
+	struct list_head	exec_head;
 	char			*hostname;
-	char			*tag;
+	char			*share_tag;
-	char			*channel;
 	int			init_pid;
-	uint32_t		c_num;
 	uint32_t		i_num;
 	uint32_t		r_num;
 	uint32_t		e_num;
+	uint32_t		d_num;
 	uint32_t		type;
-	uint32_t		code;
+	/* how many containers are running */
 	uint32_t		remains;
 	uint8_t			policy;
 	int			efd;
 	struct hyper_event	sig;
-	struct hyper_event	ctl;
 };
 
 struct hyper_win_size {
@@ -63,17 +66,35 @@ struct hyper_win_size {
 	uint64_t	seq;
 };
 
+struct hyper_killer {
+	char		*id;
+	int		signal;
+};
+
+struct hyper_reader {
+	char		*id;
+	char		*file;
+};
+
+struct hyper_writter {
+	char		*id;
+	char		*file;
+	uint8_t		*data;
+	int		len;
+};
+
 struct hyper_ctl {
 	int			efd;
 	struct hyper_event	sig;
 	struct hyper_event	tty;
 	struct hyper_event	chan;
-	struct hyper_event	ctl;
 };
 
 int hyper_mkdir(char *hyper_path);
 int hyper_open_serial(char *tty);
 struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id);
+int hyper_start_containers(struct hyper_pod *pod);
+void hyper_cleanup_pod(struct hyper_pod *pod);
 
 extern struct hyper_pod global_pod;
 extern struct hyper_ctl ctl;

src/jsmn.c

@@ -164,14 +164,18 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
 	int i;
 	jsmntok_t *token;
 	int count = 0;
+	int num = 0;
+	int out = 0;
 
-	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
+	for (; parser->pos < len && js[parser->pos] != '\0' && !out; parser->pos++) {
 		char c;
 		jsmntype_t type;
 
 		c = js[parser->pos];
 		switch (c) {
-		case '{': case '[':
+		case '{':
+			num++;
+		case '[':
 			count++;
 			if (tokens == NULL)
 				break;
@@ -189,7 +193,10 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
 			token->start = parser->pos;
 			parser->toksuper = parser->toknext - 1;
 			break;
-		case '}': case ']':
+		case '}':
+			if (--num <= 0)
+				out = 1;
+		case ']':
 			if (tokens == NULL)
 				break;
 			type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
@@ -221,7 +228,7 @@ jsmnerr_t jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
 
 					parser->toksuper = -1;
 					token->end = parser->pos + 1;
-						break;
+					break;
 				}
 			}
 			/* Error if unmatched closing bracket */

src/net.c

@@ -10,6 +10,7 @@
 #include <termios.h>
 
 #include "hyper.h"
+#include "util.h"
 #include "../config.h"
 
 void hyper_set_be32(uint8_t *buf, uint32_t val)
@@ -90,7 +91,7 @@ int hyper_send_type(int fd, uint32_t type)
 	return hyper_send_msg(fd, type, 0, NULL);
 }
 
-int hyper_get_type_block(int fd, uint32_t *type)
+int hyper_get_type(int fd, uint32_t *type)
 {
 	int len = 0, size;
 	uint8_t buf[8];
@@ -111,42 +112,20 @@ int hyper_get_type_block(int fd, uint32_t *type)
 	return 0;
 }
 
-int hyper_send_type_block(int fd, uint32_t type, int need_ack)
+int hyper_send_msg_block(int fd, uint32_t type, uint32_t len, uint8_t *data)
 {
-	int ret = 0, flags;
+	int ret, flags;
-	uint32_t t;
 
-	flags = fcntl(fd, F_GETFL, 0);
+	flags = hyper_setfd_block(fd);
 	if (flags < 0) {
-		fprintf(stderr, "get fd flag failed\n");
+		fprintf(stderr, "%s fail to set fd block\n", __func__);
 		return -1;
 	}
 
-	if (fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) < 0) {
+	ret = hyper_send_msg(fd, type, len, data);
-		perror("set fd BLOCK failed");
-		return -1;
-	}
 
-	ret = hyper_send_msg(fd, type, 0, NULL);
+	if (fcntl(fd, F_SETFL, flags) < 0) {
-	if (ret < 0)
+		perror("restore fd flag failed");
-		goto out;
-
-	if (need_ack == 0)
-		goto out;
-
-	ret = hyper_get_type_block(fd, &t);
-	if (ret < 0) {
-		fprintf(stderr, "can not get type\n");
-		goto out;
-	}
-
-	fprintf(stdout, "get type %" PRIu32"\n", type);
-
-	if (t != ACK)
-		ret = -1;
-out:
-	if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) {
-		perror("set fd BLOCK failed");
 		return -1;
 	}
 
@@ -307,14 +286,17 @@ static int hyper_remove_nic(char *device)
 {
 	char path[256], real[128];
 	int fd;
+	ssize_t size;
 
 	sprintf(path, "/sys/class/net/%s", device);
 
-	if (readlink(path, real, 128) < 0) {
+	size = readlink(path, real, 128);
+	if (size < 0 || size > 127) {
 		perror("fail to read link directory");
 		return -1;
 	}
 
+	real[size] = '\0';
 	sprintf(path, "/sys/%s/../../../remove", real + 5);
 
 	fprintf(stdout, "get net sys path %s\n", path);
@@ -676,6 +658,11 @@ static int hyper_cleanup_interface(struct rtnl_handle *rth,
 		return -1;
 	}
 
+	/* Don't down&remove lo device */
+	if (strcmp(iface->device, "lo") == 0) {
+		return 0;
+	}
+
 	if (hyper_down_nic(rth, iface->ifindex) < 0) {
 		fprintf(stderr, "up device %d failed\n", iface->ifindex);
 		return -1;
@@ -795,3 +782,64 @@ void hyper_cleanup_network(struct hyper_pod *pod)
 	pod->i_num = 0;
 	netlink_close(&rth);
 }
+
+int hyper_setup_dns(struct hyper_pod *pod)
+{
+	int i, fd, ret = -1;
+	char buf[28];
+
+	if (pod->dns == NULL)
+		return 0;
+
+	fd = open("/tmp/hyper/resolv.conf", O_CREAT| O_TRUNC| O_WRONLY, 0644);
+
+	if (fd < 0) {
+		perror("create /tmp/resolv.conf failed");
+		return -1;
+	}
+
+	for (i = 0; i < pod->d_num; i++) {
+		int size = snprintf(buf, sizeof(buf), "nameserver %s\n", pod->dns[i]);
+		int len = 0, l;
+
+		if (size < 0) {
+			fprintf(stderr, "sprintf resolv.conf entry failed\n");
+			goto out;
+		}
+
+		while (len < size) {
+			l = write(fd, buf + len, size - len);
+			if (l < 0) {
+				perror("fail to write resolv.conf");
+				goto out;
+			}
+			len += l;
+		}
+	}
+
+	ret = 0;
+out:
+	close(fd);
+	return ret;
+}
+
+void hyper_cleanup_dns(struct hyper_pod *pod)
+{
+	int fd, i;
+
+	for (i = 0; i < pod->d_num; i++) {
+		free(pod->dns[i]);
+	}
+
+	free(pod->dns);
+	pod->dns = NULL;
+	pod->d_num = 0;
+
+	fd = open("/tmp/hyper/resolv.conf", O_WRONLY| O_TRUNC);
+	if (fd < 0) {
+		perror("open /tmp/hyper/resolv.conf failed");
+		return;
+	}
+
+	close(fd);
+}

src/net.h

@@ -48,10 +48,12 @@ void hyper_set_be64(uint8_t *buf, uint64_t val);
 uint64_t hyper_get_be64(uint8_t *buf);
 int hyper_setup_network(struct hyper_pod *pod);
 void hyper_cleanup_network(struct hyper_pod *pod);
-int hyper_get_type_block(int fd, uint32_t *type);
+int hyper_setup_dns(struct hyper_pod *pod);
+void hyper_cleanup_dns(struct hyper_pod *pod);
+int hyper_get_type(int fd, uint32_t *type);
 int hyper_send_type(int fd, uint32_t type);
 int hyper_send_type_block(int fd, uint32_t type, int need_ack);
-int hyper_send_msg(int fd, uint32_t type, uint32_t len,
+int hyper_send_msg(int fd, uint32_t type, uint32_t len, uint8_t *data);
-		 uint8_t *message);
+int hyper_send_msg_block(int fd, uint32_t type, uint32_t len, uint8_t *data);
 int hyper_send_data(int fd, uint8_t *data, uint32_t len);
 #endif

src/parse.h

@@ -1,5 +1,5 @@
-#ifndef _DVM_JSON_H_
+#ifndef _PARSE_H_
-#define _DVM_JSON_H_
+#define _PARSE_H_
 
 #include "hyper.h"
 #include "jsmn.h"
@@ -9,5 +9,10 @@ struct hyper_exec *hyper_parse_execcmd(char *json, int length);
 char *json_token_str(char *js, jsmntok_t *t);
 int json_token_streq(char *js, jsmntok_t *t, char *s);
 int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length);
+int hyper_parse_kill_container(struct hyper_killer *killer, char *json, int length);
+int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length);
+int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length);
+struct hyper_container *hyper_parse_new_container(struct hyper_pod *pod, char *json, int length);
+void hyper_free_container(struct hyper_container *c);
 
 #endif

src/syscall.h

@@ -0,0 +1,16 @@
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <sys/syscall.h>
+
+/*
+ * Use raw syscall for versions of glibc that don't include it. But this
+ * requires kernel-headers for syscall number hint.
+ */
+
+#if !defined(HAVE_SETNS) && defined(__NR_setns)
+static inline int setns(int fd, int nstype)
+{
+	errno = syscall(__NR_setns, fd, nstype);
+	return errno == 0 ? 0 : -1;
+}
+#endif

src/util.c

@@ -11,11 +11,13 @@
 #include <ctype.h>
 #include <mntent.h>
 #include <sys/mount.h>
+#include <sys/socket.h>
 #include <sys/reboot.h>
 #include <linux/reboot.h>
 
 #include "util.h"
 #include "hyper.h"
+#include "container.h"
 #include "../config.h"
 
 char *read_cmdline(void)
@@ -23,6 +25,22 @@ char *read_cmdline(void)
 	return NULL;
 }
 
+int hyper_setup_env(struct env *envs, int num)
+{
+	int i, ret = 0;
+	struct env *env;
+
+	for (i = 0; i < num; i++) {
+		env = &envs[i];
+		if (setenv(env->env, env->value, 1) < 0) {
+			perror("fail to setup env");
+			ret = -1;
+		}
+	}
+
+	return ret;
+}
+
 int hyper_list_dir(char *path)
 {
 	struct dirent **list;
@@ -39,12 +57,47 @@ int hyper_list_dir(char *path)
 	for (i = 0; i < num; i++) {
 		dir = list[i];
 		fprintf(stdout, "%s get %s\n", path, dir->d_name);
+		free(dir);
 	}
 
 	free(list);
 	return 0;
 }
 
+int hyper_find_sd(char *prefix, char *addr, char **dev) {
+	struct dirent **list;
+	struct dirent *dir;
+	char path[512];
+	int i, num;
+
+	sprintf(path, "%s/sys/class/scsi_disk/0:0:%s/device/block/", prefix, addr);
+	fprintf(stdout, "orig dev %s, scan path %s\n", *dev, path);
+
+	num = scandir(path, &list, NULL, NULL);
+	if (num < 0) {
+		perror("scan path failed");
+		return -1;
+	}
+
+	for (i = 0; i < num; i++) {
+		dir = list[i];
+		if (dir->d_name[0] == '.') {
+			continue;
+		}
+
+		fprintf(stdout, "%s get %s\n", path, dir->d_name);
+		free(*dev);
+		*dev = strdup(dir->d_name);
+		break;
+	}
+
+	for (i = 0; i < num; i++)
+		free(list[i]);
+
+	free(list);
+	return 0;
+}
+
 int hyper_mkdir(char *hyper_path)
 {
 	struct stat st;
@@ -174,8 +227,7 @@ int hyper_insmod(char *module)
 	ret = 0;
 out:
 	close(fd);
-	if (buf)
+	free(buf);
-		free(buf);
 
 	return ret;
 err:
@@ -286,7 +338,7 @@ int hyper_setfd_block(int fd)
 		return -1;
 	}
 
-	return 0;
+	return flags;
 }
 
 int hyper_setfd_nonblock(int fd)
@@ -303,10 +355,25 @@ int hyper_setfd_nonblock(int fd)
 		return -1;
 	}
 
+	return flags;
+}
+
+int hyper_socketpair(int domain, int type, int protocol, int sv[2])
+{
+	if (socketpair(domain, type, protocol, sv) < 0) {
+		perror("socketpair failed");
+		return -1;
+	}
+
+	if (hyper_setfd_cloexec(sv[0]) < 0 ||
+	    hyper_setfd_cloexec(sv[1]) < 0) {
+		return -1;
+	}
+
 	return 0;
 }
 
-void hyper_unmount_all(void)
+static void hyper_unmount_all(void)
 {
 	FILE *mtab;
 	struct mntent *mnt;
@@ -351,64 +418,9 @@ void hyper_unmount_all(void)
 	sync();
 }
 
-void hyper_kill_all(void)
+void hyper_shutdown()
 {
-	int npids = 0;
+	hyper_send_msg_block(ctl.chan.fd, ACK, 0, NULL);
-	int index = 0;
-	int pid;
-	DIR *dp;
-	struct dirent *de;
-	pid_t *pids = NULL;
-
-	dp = opendir("/proc");
-	if (dp == NULL)
-		return;
-
-	while ((de = readdir(dp)) && de != NULL) {
-		if (!isdigit(de->d_name[0]))
-			continue;
-		pid = atoi(de->d_name);
-		if (pid == 1)
-			continue;
-		if (index <= npids) {
-			pids = realloc(pids, npids + 16384);
-			if (pids == NULL)
-				return;
-			npids += 16384;
-		}
-
-		pids[index++] = pid;
-	}
-
-	fprintf(stdout, "Sending SIGTERM\n");
-
-	for (--index; index >= 0; --index) {
-		fprintf(stdout, "kill process %d\n", pids[index]);
-		kill(pids[index], SIGTERM);
-	}
-
-	free(pids);
-	closedir(dp);
-}
-
-int hyper_send_finish(struct hyper_pod *pod)
-{
-	int i;
-	uint8_t *data = calloc(pod->c_num, 4);
-
-	for (i = 0; i < pod->c_num; i++)
-		hyper_set_be32(data + (i * 4), pod->c[i].exec.code);
-
-	return hyper_send_msg(ctl.chan.fd, FINISH, pod->c_num * 4, data);
-}
-
-void hyper_shutdown(struct hyper_pod *pod)
-{
-	hyper_send_finish(pod);
-
-	hyper_kill_all();
-
 	hyper_unmount_all();
-
 	reboot(LINUX_REBOOT_CMD_POWER_OFF);
 }

src/util.h

@@ -5,6 +5,7 @@
 #include "../config.h"
 
 struct hyper_pod;
+struct env;
 
 #ifdef WITH_DEBUG
 #define dprintf(fmt, ...) \
@@ -14,6 +15,8 @@ struct hyper_pod;
 #endif
 
 char *read_cmdline(void);
+int hyper_setup_env(struct env *envs, int num);
+int hyper_find_sd(char *prefix, char *addr, char **dev);
 int hyper_list_dir(char *path);
 int hyper_mkdir(char *path);
 int hyper_open_channel(char *channel, int mode);
@@ -21,9 +24,7 @@ int hyper_open_serial_dev(char *tty);
 int hyper_setfd_cloexec(int fd);
 int hyper_setfd_block(int fd);
 int hyper_setfd_nonblock(int fd);
-void hyper_shutdown(struct hyper_pod *pod);
+int hyper_socketpair(int domain, int type, int protocol, int sv[2]);
-int hyper_send_finish(struct hyper_pod *pod);
+void hyper_shutdown(void);
-void hyper_kill_all(void);
-void hyper_unmount_all(void);
 int hyper_insmod(char *module);
 #endif