Merge pull request #222 from laijs/event-hanghup

change policy for event hangup

.gitignore

@@ -7,6 +7,8 @@
 .#*
 .git
 .deps
+TAGS
+tags
 init
 build/hyper_daemon
 build/hyper-initrd.img

.travis.yml

@@ -0,0 +1,24 @@
+sudo: required
+dist: trusty
+
+language: c
+
+matrix:
+  include:
+    - go: 1.7
+
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -y autoconf automake pkg-config libdevmapper-dev libsqlite3-dev libvirt-dev qemu libvirt-bin -qq
+  - cd `mktemp -d`
+  - wget https://git.fedorahosted.org/cgit/lvm2.git/snapshot/lvm2-2_02_131.tar.xz
+  - tar xf lvm2-2_02_131.tar.xz
+  - cd lvm2-2_02_131
+  - ./configure && make device-mapper && sudo make install
+
+script:
+  - cd ${TRAVIS_BUILD_DIR}
+  - ./autogen.sh
+  - ./configure
+  - make
+  - hack/test-cmd.sh

README.md

@@ -12,6 +12,11 @@ clone this repo, and make sure you have build-essentials installed. Go into the
 
 Then you can find `hyper-initrd.img` in the build directory, together with a pre-built kernel.
 
+If you want to run hyperstart with 64-bit ARM architecture, please reconfigure with flag --with-aarch64,
+
+    > ./configure --with-aarch64
+    > make
+
 If you want to get the boot disk file for VirtualBox, please reconfigure with flag --with-vbox,
 
     > ./configure --with-vbox

autogen.sh

@@ -8,34 +8,34 @@ DIE=0
 
 test -f src/init.c || {
 	echo
-	echo "You must run this script in the top-level hyperint drectory."
+	echo "You must run this script in the top-level hyperstart drectory."
 	echo
 	DIE=1
 }
 
 (autoconf --version) < /dev/null > /dev/null 2>&1 || {
 	echo
-	echo "You must have autoconf installed to generate the hyperinit."
+	echo "You must have autoconf installed to generate the hyperstart."
 	echo
 	DIE=1
 }
 
 (autoheader --version) < /dev/null > /dev/null 2>&1 || {
 	echo
-	echo "You must have autoheader installed to generate the hypernit."
+	echo "You must have autoheader installed to generate the hyperstart."
 	echo
 	DIE=1
 }
 
 (automake --version) < /dev/null > /dev/null 2>&1 || {
 	echo
-	echo "You must have automake installed to generate the hypernit."
+	echo "You must have automake installed to generate the hyperstart."
 	echo
 	DIE=1
 }
 (autoreconf --version) < /dev/null > /dev/null 2>&1 || {
 	echo
-	echo "You must have autoreconf installed to generate the hypernit."
+	echo "You must have autoreconf installed to generate the hyperstart."
 	echo
 	DIE=1
 }

build/Makefile.am

@@ -4,9 +4,14 @@ if WITH_VBOX
 initrd-local:
 	bash ./make-initrd.sh vbox
 else
+if WITH_AARCH64
+initrd-local:
+	bash ./make-initrd.sh aarch64
+else
 initrd-local:
 	bash ./make-initrd.sh
 endif
+endif
 
 cbfs-local:
 	bash ./make-initrd.sh cbfs

build/make-initrd.sh

@@ -1,13 +1,41 @@
 #!/bin/bash
 
 rm -rf /tmp/hyperstart-rootfs
-mkdir -p /tmp/hyperstart-rootfs/lib /tmp/hyperstart-rootfs/lib64 /tmp/hyperstart-rootfs/lib/modules
+mkdir -p /tmp/hyperstart-rootfs/lib \
+	  /tmp/hyperstart-rootfs/lib64 \
+	  /tmp/hyperstart-rootfs/lib/modules
+
+mkdir -m 0755 -p /tmp/hyperstart-rootfs/dev \
+	  /tmp/hyperstart-rootfs/sys \
+	  /tmp/hyperstart-rootfs/sbin \
+	  /tmp/hyperstart-rootfs/bin \
+	  /tmp/hyperstart-rootfs/proc
 
 cp ../src/init /tmp/hyperstart-rootfs
 cp busybox /tmp/hyperstart-rootfs
 cp iptables /tmp/hyperstart-rootfs
 cp libm.so.6 /tmp/hyperstart-rootfs/lib64/
-tar -xf modules.tar -C /tmp/hyperstart-rootfs/lib/modules
+
+if [ "$1"x = "aarch64"x ]; then
+	echo "build hyperstart for aarch64"
+	tar -xf modules_aarch64.tar -C /tmp/hyperstart-rootfs/lib/modules
+else
+	tar -xf modules.tar -C /tmp/hyperstart-rootfs/lib/modules
+fi
+
+# create symlinks to busybox and iptables
+BUSYBOX_BINARIES=(/bin/sh /bin/tar /bin/hwclock /sbin/modprobe /sbin/depmod)
+for bin in ${BUSYBOX_BINARIES[@]}
+do
+	mkdir -p /tmp/hyperstart-rootfs/`dirname ${bin}`
+	ln -sf /busybox /tmp/hyperstart-rootfs/${bin}
+done
+IPTABLES_BINARIES=(/sbin/iptables /sbin/iptables-restore /sbin/iptables-save)
+for bin in ${IPTABLES_BINARIES[@]}
+do
+	mkdir -p /tmp/hyperstart-rootfs/`dirname ${bin}`
+	ln -sf /iptables /tmp/hyperstart-rootfs/${bin}
+done
 
 ldd /tmp/hyperstart-rootfs/init | while read line
 do
@@ -30,7 +58,6 @@ fi
 
 ( cd /tmp/hyperstart-rootfs && find . | cpio -H newc -o | gzip -9 ) > ./hyper-initrd.img
 
-cd ../
 rm -rf /tmp/hyperstart-rootfs
 
 if [ "$1"x = "cbfs"x ]; then

configure.ac

@@ -38,6 +38,17 @@ if test "$fail" = "1" ; then
     AC_MSG_ERROR(Unable to find necessary functions)
 fi
 
+AC_ARG_WITH([aarch64],
+            [AS_HELP_STRING([--with-aarch64],
+                            [run hyperstart with 64-bit ARM architecture])],
+            [],[with_aarch64=no])
+
+if test "x$with_aarch64" != "xno" ; then
+    AC_DEFINE_UNQUOTED([WITH_AARCH64], 1, [run hyperstart with 64-bit ARM architecture])
+fi
+
+AM_CONDITIONAL([WITH_AARCH64], [test "x$with_aarch64" != "xno"])
+
 AC_ARG_WITH([vbox],
             [AS_HELP_STRING([--with-vbox],
                             [run hyperstart on Virtualbox])],
@@ -61,6 +72,7 @@ AC_MSG_RESULT([
 	prefix:		${prefix}
 
 	with-vbox:	${with_vbox}
+	with-aarch64:	${with_aarch64}
 
 	compiler:          ${CC}
 	cflags:            ${CFLAGS}

hack/test-cmd.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+###########################
+# test hyperstart from hyper
+
+export HYPER_RUNTIME=/var/lib/hyper
+sudo mkdir -p ${HYPER_RUNTIME}
+sudo cp build/kernel ${HYPER_RUNTIME}/kernel
+sudo cp build/hyper-initrd.img ${HYPER_RUNTIME}/hyper-initrd.img
+
+mkdir -p ${GOPATH}/src/github.com/hyperhq
+git clone https://github.com/hyperhq/hyperd.git ${GOPATH}/src/github.com/hyperhq/hyperd
+cd ${GOPATH}/src/github.com/hyperhq/hyperd
+./autogen.sh
+./configure
+make
+hack/test-cmd.sh
+

src/Makefile.am

@@ -1,3 +1,3 @@
-AM_CFLAGS = -Wall
+AM_CFLAGS = -Wall -Werror
 bin_PROGRAMS=init
 init_SOURCES=init.c jsmn.c net.c util.c parse.c parson.c container.c exec.c event.c portmapping.c

src/api.h

@@ -0,0 +1,51 @@
+#ifndef _HYPERSTART_API_H_
+#define _HYPERSTART_API_H_
+
+#define APIVERSION 4242
+
+// control command id
+enum {
+	GETVERSION,
+	STARTPOD,
+	GETPOD,
+	STOPPOD_DEPRECATED,
+	DESTROYPOD,
+	RESTARTCONTAINER,
+	EXECCMD,
+	CMDFINISHED,
+	READY,
+	ACK,
+	ERROR,
+	WINSIZE,
+	PING,
+	PODFINISHED,
+	NEXT,
+	WRITEFILE,
+	READFILE,
+	NEWCONTAINER,
+	KILLCONTAINER,
+	ONLINECPUMEM,
+	SETUPINTERFACE,
+	SETUPROUTE,
+	REMOVECONTAINER,
+};
+
+/*
+ * control message format
+ * | ctrl id | length  | payload (length-8)      |
+ * | . . . . | . . . . | . . . . . . . . . . . . |
+ * 0         4         8                         length
+ */
+#define CONTROL_HEADER_SIZE		8
+#define CONTROL_HEADER_LENGTH_OFFSET	4
+
+/*
+ * stream message format
+ * | stream sequence | length  | payload (length-12)     |
+ * | . . . . . . . . | . . . . | . . . . . . . . . . . . |
+ * 0                 8         12                        length
+ */
+#define STREAM_HEADER_SIZE		12
+#define STREAM_HEADER_LENGTH_OFFSET	8
+
+#endif /* _HYPERSTART_API_H_ */

src/container.c

@@ -29,7 +29,7 @@ static int container_populate_volume(char *src, char *dest)
 
 	if (stat(dest, &st) == 0) {
 		if (!S_ISDIR(st.st_mode)) {
-			fprintf(stderr, "the _data in volume %s is not directroy\n", dest);
+			fprintf(stderr, "the _data in volume %s is not directory\n", dest);
 			return -1;
 		}
 
@@ -42,7 +42,7 @@ static int container_populate_volume(char *src, char *dest)
 	}
 
 	if (hyper_mkdir(dest, 0777) < 0) {
-		fprintf(stderr, "fail to create directroy %s\n", dest);
+		fprintf(stderr, "fail to create directory %s\n", dest);
 		return -1;
 	}
 
@@ -158,7 +158,7 @@ static int container_setup_volume(struct hyper_container *container)
 			sprintf(volume, "/%s/_data/%s", path, filevolume);
 			/* 0777 so that any user can read/write the new file volume */
 			if (chmod(volume, 0777) < 0) {
-				fprintf(stderr, "fail to chmod directroy %s\n", volume);
+				fprintf(stderr, "fail to chmod directory %s\n", volume);
 				return -1;
 			}
 		}
@@ -183,7 +183,7 @@ static int container_setup_volume(struct hyper_container *container)
 		struct fsmap *map = &container->maps[i];
 		char mountpoint[512];
 
-		sprintf(path, "/tmp/hyper/shared/%s", map->source);
+		sprintf(path, "%s/%s", SHARED_DIR, map->source);
 		sprintf(mountpoint, "./%s", map->path);
 		fprintf(stdout, "mount %s to %s\n", path, mountpoint);
 
@@ -313,15 +313,23 @@ static int container_setup_mount(struct hyper_container *container)
 		return -1;
 	}
 
-	if (unlink("./dev/ptmx") < 0)
+	if (unlink("./dev/ptmx") < 0) {
 		perror("remove /dev/ptmx failed");
-	if (symlink("/dev/pts/ptmx", "./dev/ptmx") < 0)
+		return -1;
+	}
+	if (symlink("/dev/pts/ptmx", "./dev/ptmx") < 0) {
 		perror("link /dev/pts/ptmx to /dev/ptmx failed");
+		return -1;
+	}
 
-	symlink("/proc/self/fd", "./dev/fd");
-	symlink("/proc/self/fd/0", "./dev/stdin");
-	symlink("/proc/self/fd/1", "./dev/stdout");
-	symlink("/proc/self/fd/2", "./dev/stderr");
+	/* all containers share the same devtmpfs, so we need to ignore the errno EEXIST */
+	if ((symlink("/proc/self/fd", "./dev/fd") < 0 && errno != EEXIST) ||
+	    (symlink("/proc/self/fd/0", "./dev/stdin") < 0 && errno != EEXIST) ||
+	    (symlink("/proc/self/fd/1", "./dev/stdout") < 0 && errno != EEXIST) ||
+	    (symlink("/proc/self/fd/2", "./dev/stderr") < 0 && errno != EEXIST)) {
+		perror("failed to symlink for /dev/fd, /dev/stdin, /dev/stdout or /dev/stderr");
+		return -1;
+	}
 
 	return 0;
 }
@@ -456,7 +464,7 @@ static int hyper_rescan_scsi(void)
 
 	num = scandir("/sys/class/scsi_host/", &list, NULL, NULL);
 	if (num < 0) {
-		perror("scan /sys/calss/virtio-ports/ failed");
+		perror("scan /sys/class/scsi_host/ failed");
 		return -1;
 	}
 
@@ -507,12 +515,21 @@ static int hyper_setup_container_rootfs(void *data)
 	int setup_dns;
 	uint32_t type;
 
+	/* wait for ns-opened ready message */
+	if (hyper_get_type(arg->pipens[0], &type) < 0 || type != READY) {
+		fprintf(stderr, "wait for /proc/self/ns/mnt opened failed\n");
+		goto fail;
+	}
+
 	if (hyper_enter_sandbox(arg->pod, -1) < 0) {
 		perror("enter sandbox failed");
 		goto fail;
 	}
 
-	if (hyper_rescan_scsi() < 0) {
+	/* To create files/directories accessible for all users. */
+	umask(0);
+
+	if (container->fstype && hyper_rescan_scsi() < 0) {
 		fprintf(stdout, "rescan scsi failed\n");
 		goto fail;
 	}
@@ -529,7 +546,7 @@ static int hyper_setup_container_rootfs(void *data)
 
 	sprintf(root, "/tmp/hyper/%s/root/", container->id);
 	if (hyper_mkdir(root, 0755) < 0) {
-		perror("make root directroy failed");
+		perror("make root directory failed");
 		goto fail;
 	}
 
@@ -558,7 +575,7 @@ static int hyper_setup_container_rootfs(void *data)
 	} else {
 		char path[512];
 
-		sprintf(path, "/tmp/hyper/shared/%s/", container->image);
+		sprintf(path, "%s/%s/", SHARED_DIR, container->image);
 		fprintf(stdout, "src directory %s\n", path);
 
 		if (mount(path, root, NULL, MS_BIND, NULL) < 0) {
@@ -575,7 +592,10 @@ static int hyper_setup_container_rootfs(void *data)
 		perror("failed to bind rootfs");
 		goto fail;
 	}
-	chdir(rootfs);
+	if (chdir(rootfs) < 0) {
+		perror("failed to change the root to path of the container root(before manipulating)");
+		goto fail;
+	}
 
 	/*
 	 * Recreate dns resolver iif configured by pod spec. Other cases
@@ -612,9 +632,15 @@ static int hyper_setup_container_rootfs(void *data)
 	}
 	/* pivot_root won't work, see
 	 * Documention/filesystem/ramfs-rootfs-initramfs.txt */
-	chroot(".");
+	if (chroot(".") < 0) {
+		perror("failed to setup the root for the mount namepsace");
+		goto fail;
+	}
 
-	chdir("/");
+	if (chdir("/") < 0) {
+		perror("failed chdir to the new root");
+		goto fail;
+	}
 
 	if (container_setup_sysctl(container) < 0) {
 		fprintf(stderr, "container sets up sysctl failed\n");
@@ -626,12 +652,6 @@ static int hyper_setup_container_rootfs(void *data)
 		goto fail;
 	}
 
-	/* wait for ns-opened ready message */
-	if (hyper_get_type(arg->pipens[0], &type) < 0 || type != READY) {
-		fprintf(stderr, "wait for /proc/self/ns/mnt opened failed\n");
-		goto fail;
-	}
-
 	hyper_send_type(arg->pipe[1], READY);
 	fflush(NULL);
 	_exit(0);
@@ -648,7 +668,7 @@ static int hyper_setup_pty(struct hyper_container *c)
 	sprintf(root, "/tmp/hyper/%s/devpts/", c->id);
 
 	if (hyper_mkdir(root, 0755) < 0) {
-		perror("make container pts directroy failed");
+		perror("make container pts directory failed");
 		return -1;
 	}
 
@@ -676,6 +696,8 @@ int hyper_setup_container(struct hyper_container *container, struct hyper_pod *p
 	uint32_t type;
 	int pid;
 
+	container->exec.pod = pod;
+
 	if (pipe2(arg.pipe, O_CLOEXEC) < 0 || pipe2(arg.pipens, O_CLOEXEC) < 0) {
 		perror("create pipe between pod init execcmd failed");
 		goto fail;

src/event.c

@@ -116,98 +116,6 @@ int hyper_requeue_event(int efd, struct hyper_event *ev)
 	return 0;
 }
 
-static int hyper_getmsg_len(struct hyper_event *he, uint32_t *len)
-{
-	struct hyper_buf *buf = &he->rbuf;
-
-	if (buf->get < he->ops->len_offset + 4)
-		return -1;
-
-	*len = hyper_get_be32(buf->data + he->ops->len_offset);
-	return 0;
-}
-
-int hyper_event_read(struct hyper_event *he, int efd)
-{
-	struct hyper_buf *buf = &he->rbuf;
-	uint32_t len = 4;
-	uint8_t data[4];
-	int offset = he->ops->len_offset;
-	int end = offset + 4;
-	int size;
-
-	fprintf(stdout, "%s\n", __func__);
-
-	while (hyper_getmsg_len(he, &len) < 0) {
-		size = read(he->fd, buf->data + buf->get, end - buf->get);
-		if (size > 0) {
-			buf->get += size;
-			fprintf(stdout, "already read %" PRIu32 " bytes data\n",
-				buf->get);
-
-			if (he->ops->ack) {
-				/* control channel, need ack */
-				hyper_set_be32(data, size);
-				hyper_send_msg(he->fd, NEXT, 4, data);
-			}
-			continue;
-		}
-
-		if (errno == EINTR)
-			continue;
-
-		if (errno != EAGAIN && size != 0) {
-			perror("fail to read");
-			return -1;
-		}
-
-		return 0;
-	}
-
-	fprintf(stdout, "get length %" PRIu32"\n", len);
-	if (len > buf->size) {
-		fprintf(stderr, "get length %" PRIu32", too long\n", len);
-		return -1;
-	}
-
-	while (buf->get < len) {
-		size = read(he->fd, buf->data + buf->get, len - buf->get);
-		if (size > 0) {
-			buf->get += size;
-			fprintf(stdout, "read %d bytes data, total data %" PRIu32 "\n",
-				size, buf->get);
-			if (he->ops->ack) {
-				/* control channel, need ack */
-				hyper_set_be32(data, size);
-				hyper_send_msg(he->fd, NEXT, 4, data);
-			}
-
-			continue;
-		}
-
-		if (errno == EINTR)
-			continue;
-
-		if (errno != EAGAIN && size != 0) {
-			perror("fail to read");
-			return -1;
-		}
-
-		/* size == 0 : No one connect to qemu socket */
-		return 0;
-	}
-
-	/* get the whole data */
-	if (he->ops->handle(he, len) != 0)
-		return -1;
-
-	/* len: length of the already get new data */
-	buf->get -= len;
-	memmove(buf->data, buf->data + len, buf->get);
-
-	return 0;
-}
-
 int hyper_event_write(struct hyper_event *he, int efd)
 {
 	struct hyper_buf *buf = &he->wbuf;
@@ -230,9 +138,7 @@ int hyper_event_write(struct hyper_event *he, int efd)
 	memmove(buf->data, buf->data + len, buf->get);
 
 	if (buf->get == 0) {
-		hyper_modify_event(ctl.efd, he, he->flag & ~(EPOLLOUT| EPOLLPRI));
-	} else if (!FULL(buf)) {
-		hyper_modify_event(ctl.efd, he, he->flag & ~EPOLLPRI);
+		hyper_modify_event(ctl.efd, he, he->flag & ~EPOLLOUT);
 	}
 
 	return 0;
@@ -251,32 +157,23 @@ int hyper_handle_event(int efd, struct epoll_event *event)
 	fprintf(stdout, "%s get event %d, he %p, fd %d. ops %p\n",
 			__func__, event->events, he, he->fd, he->ops);
 
-	/* do not handle hup event if have in event */
+	/* do not handle hup event if have in/out event */
 	if ((event->events & EPOLLIN) && he->ops->read) {
 		fprintf(stdout, "%s event EPOLLIN, he %p, fd %d, %p\n",
 			__func__, he, he->fd, he->ops);
-		if (he->ops->read && he->ops->read(he, efd) < 0)
-			return -1;
-	} else if (event->events & EPOLLHUP) {
-		fprintf(stdout, "%s event EPOLLHUP, he %p, fd %d, %p\n",
-			__func__, he, he->fd, he->ops);
-		if (he->ops->hup)
-			he->ops->hup(he, efd);
-		return 0;
+		return he->ops->read(he, efd);
 	}
-
-	if (event->events & EPOLLOUT) {
+	if ((event->events & EPOLLOUT) && he->ops->write) {
 		fprintf(stdout, "%s event EPOLLOUT, he %p, fd %d, %p\n",
 			__func__, he, he->fd, he->ops);
-		if (he->ops->write && he->ops->write(he, efd) < 0)
-			return -1;
+		return he->ops->write(he, efd);
 	}
 
-	if (event->events & EPOLLERR) {
-		fprintf(stderr, "get epoll err of not epool in event\n");
+	if ((event->events & EPOLLHUP) || (event->events & EPOLLERR)) {
+		fprintf(stdout, "%s event EPOLLHUP or EPOLLERR, he %p, fd %d, %x\n",
+			__func__, he, he->fd, event->events);
 		if (he->ops->hup)
 			he->ops->hup(he, efd);
-		return 0;
 	}
 
 	return 0;

src/event.h

@@ -9,12 +9,9 @@ struct hyper_event;
 struct hyper_event_ops {
 	int		(*read)(struct hyper_event *e, int efd);
 	int		(*write)(struct hyper_event *e, int efd);
-	int		(*handle)(struct hyper_event *e, uint32_t len);
 	void		(*hup)(struct hyper_event *e, int efd);
 	int		rbuf_size;
 	int		wbuf_size;
-	int		len_offset;
-	int		ack;
 };
 
 struct hyper_buf {
@@ -43,6 +40,5 @@ int hyper_init_event(struct hyper_event *de, struct hyper_event_ops *ops,
 int hyper_handle_event(int efd, struct epoll_event *event);
 void hyper_reset_event(struct hyper_event *de);
 void hyper_event_hup(struct hyper_event *de, int efd);
-int hyper_event_read(struct hyper_event *dei, int efd);
 int hyper_event_write(struct hyper_event *de, int efd);
 #endif

src/exec.c

@@ -23,10 +23,15 @@
 #include "parse.h"
 #include "syscall.h"
 
-static int hyper_release_exec(struct hyper_exec *, struct hyper_pod *);
-static void hyper_exec_process(struct hyper_exec *exec);
+struct stdio_config {
+	int stdinfd, stdoutfd, stderrfd;
+	int stdinevfd, stdoutevfd, stderrevfd;
+};
 
-static int send_exec_finishing(uint64_t seq, int len, int code, int block)
+static int hyper_release_exec(struct hyper_exec *);
+static void hyper_exec_process(struct hyper_exec *exec, struct stdio_config *io);
+
+static int send_exec_finishing(uint64_t seq, int len, int code)
 {
 	struct hyper_buf *buf = &ctl.tty.wbuf;
 
@@ -50,38 +55,26 @@ static int send_exec_finishing(uint64_t seq, int len, int code, int block)
 		buf->data[buf->get + 12] = code;
 
 	buf->get += len;
-	if (!block) {
-		hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT);
-		return 0;
-	}
-
-	if (hyper_setfd_block(ctl.tty.fd) < 0 ||
-	    hyper_send_data(ctl.tty.fd, buf->data, buf->get) < 0 ||
-	    hyper_setfd_nonblock(ctl.tty.fd) < 0) {
-		fprintf(stderr, "send eof failed\n");
-		return -1;
-	}
+	hyper_modify_event(ctl.efd, &ctl.tty, EPOLLIN | EPOLLOUT);
 
 	return 0;
 }
 
-static int hyper_send_exec_eof(struct hyper_exec *exec, int block) {
-	return send_exec_finishing(exec->seq, 12, -1, block);
+static int hyper_send_exec_eof(struct hyper_exec *exec) {
+	return send_exec_finishing(exec->seq, 12, -1);
 }
 
-static int hyper_send_exec_code(struct hyper_exec *exec, int block) {
-	return send_exec_finishing(exec->seq, 13, exec->code, block);
+static int hyper_send_exec_code(struct hyper_exec *exec) {
+	return send_exec_finishing(exec->seq, 13, exec->code);
 }
 
 static void pts_hup(struct hyper_event *de, int efd, struct hyper_exec *exec)
 {
-	struct hyper_pod *pod = de->ptr;
-
 	fprintf(stdout, "%s, seq %" PRIu64"\n", __func__, exec->seq);
 
 	hyper_event_hup(de, efd);
 
-	hyper_release_exec(exec, pod);
+	hyper_release_exec(exec);
 }
 
 static void stdin_hup(struct hyper_event *de, int efd)
@@ -112,24 +105,25 @@ static int pts_loop(struct hyper_event *de, uint64_t seq, int efd, struct hyper_
 	struct hyper_buf *buf = &ctl.tty.wbuf;
 
 	if (FULL(buf)) {
-		flag |= EPOLLPRI;
 		goto out;
 	}
 
 	do {
 		size = read(de->fd, buf->data + buf->get + 12, buf->size - buf->get - 12);
-		fprintf(stdout, "%s: read %d data\n", __func__, size);
 		if (size < 0) {
 			if (errno == EINTR)
 				continue;
 
 			if (errno != EAGAIN && errno != EIO) {
-				perror("fail to read tty fd");
-				return -1;
+				perror("failed to read process's stdout/stderr");
+				pts_hup(de, efd, exec);
+				return 0;
 			}
 
 			break;
 		}
+		fprintf(stdout, "%s: read %d data\n", __func__, size);
+
 		if (size == 0) { // eof
 			pts_hup(de, efd, exec);
 			return 0;
@@ -141,7 +135,6 @@ static int pts_loop(struct hyper_event *de, uint64_t seq, int efd, struct hyper_
 	} while (!FULL(buf));
 
 	if (FULL(buf)) {
-		flag |= EPOLLPRI;
 		/* del & add event to move event to tail, this gives
 		 * other event a chance to write data to wbuf of tty. */
 		hyper_requeue_event(ctl.efd, de);
@@ -160,12 +153,10 @@ static int write_to_stdin(struct hyper_event *de, int efd)
 	struct hyper_exec *exec = container_of(de, struct hyper_exec, stdinev);
 	fprintf(stdout, "%s, seq %" PRIu64"\n", __func__, exec->seq);
 
-	int ret = hyper_event_write(de, efd);
-
-	if (ret >= 0 && de->wbuf.get == 0 && exec->close_stdin_request)
+	if (hyper_event_write(de, efd) < 0 || (de->wbuf.get == 0 && exec->close_stdin_request))
 		pts_hup(de, efd, exec);
 
-	return ret;
+	return 0;
 }
 
 struct hyper_event_ops in_ops = {
@@ -271,7 +262,10 @@ static int hyper_setup_exec_user(struct hyper_exec *exec)
 	if (exec->tty) {
 		char ptmx[512];
 		sprintf(ptmx, "/dev/pts/%d", exec->ptyno);
-		chown(ptmx, uid, gid);
+		if (chown(ptmx, uid, gid) < 0) {
+			perror("failed to change the owner for the slave pty file");
+			goto fail;
+		}
 	}
 
 	// apply
@@ -300,7 +294,7 @@ fail:
 	return -1;
 }
 
-static int hyper_setup_exec_notty(struct hyper_exec *e)
+static int hyper_setup_stdio_notty(struct hyper_exec *e, struct stdio_config *io)
 {
 	if (e->errseq == 0)
 		return -1;
@@ -311,8 +305,8 @@ static int hyper_setup_exec_notty(struct hyper_exec *e)
 		return -1;
 	}
 	hyper_setfd_nonblock(inpipe[1]);
-	e->stdinev.fd = inpipe[1];
-	e->stdinfd = inpipe[0];
+	io->stdinevfd = inpipe[1];
+	io->stdinfd = inpipe[0];
 
 	int outpipe[2];
 	if (pipe2(outpipe, O_CLOEXEC) < 0) {
@@ -320,8 +314,8 @@ static int hyper_setup_exec_notty(struct hyper_exec *e)
 		return -1;
 	}
 	hyper_setfd_nonblock(outpipe[0]);
-	e->stdoutev.fd = outpipe[0];
-	e->stdoutfd = outpipe[1];
+	io->stdoutevfd = outpipe[0];
+	io->stdoutfd = outpipe[1];
 
 	int errpipe[2];
 	if (pipe2(errpipe, O_CLOEXEC) < 0) {
@@ -329,25 +323,20 @@ static int hyper_setup_exec_notty(struct hyper_exec *e)
 		return -1;
 	}
 	hyper_setfd_nonblock(errpipe[0]);
-	e->stderrev.fd = errpipe[0];
-	e->stderrfd = errpipe[1];
+	io->stderrevfd = errpipe[0];
+	io->stderrfd = errpipe[1];
 
 	return 0;
 }
 
-static int hyper_setup_exec_tty(struct hyper_exec *e)
+static int hyper_setup_stdio(struct hyper_exec *e, struct stdio_config *io)
 {
 	int unlock = 0;
 	int ptymaster;
-	char ptmx[512], path[512];
-
-	if (e->seq == 0) {
-		fprintf(stderr, "e->seq should be set\n");
-		return -1;
-	}
+	char ptmx[512];
 
 	if (!e->tty) { // don't use tty for stdio
-		return hyper_setup_exec_notty(e);
+		return hyper_setup_stdio_notty(e, io);
 	}
 
 	if (e->errseq > 0) {
@@ -357,23 +346,11 @@ static int hyper_setup_exec_tty(struct hyper_exec *e)
 			return -1;
 		}
 		hyper_setfd_nonblock(errpipe[0]);
-		e->stderrev.fd = errpipe[0];
-		e->stderrfd = errpipe[1];
+		io->stderrevfd = errpipe[0];
+		io->stderrfd = errpipe[1];
 	}
 
-	if (e->id) {
-		if (sprintf(path, "/tmp/hyper/%s/devpts/", e->id) < 0) {
-			fprintf(stderr, "get ptmx path failed\n");
-			return -1;
-		}
-	} else {
-		if (sprintf(path, "/dev/pts/") < 0) {
-			fprintf(stderr, "get ptmx path failed\n");
-			return -1;
-		}
-	}
-
-	if (sprintf(ptmx, "%s/ptmx", path) < 0) {
+	if (sprintf(ptmx, "/tmp/hyper/%s/devpts/ptmx", e->container_id) < 0) {
 		fprintf(stderr, "get ptmx path failed\n");
 		return -1;
 	}
@@ -386,102 +363,102 @@ static int hyper_setup_exec_tty(struct hyper_exec *e)
 
 	if (ioctl(ptymaster, TIOCSPTLCK, &unlock) < 0) {
 		perror("ioctl unlock ptmx device failed");
+		close(ptymaster);
 		return -1;
 	}
 
 	if (ioctl(ptymaster, TIOCGPTN, &e->ptyno) < 0) {
 		perror("ioctl get execcmd pty device failed");
+		close(ptymaster);
 		return -1;
 	}
 
-	if (sprintf(ptmx, "%s/%d", path, e->ptyno) < 0) {
-		fprintf(stderr, "get ptmx path failed\n");
-		return -1;
-	}
-
-	e->ptyfd = open(ptmx, O_RDWR | O_NOCTTY | O_CLOEXEC);
-	fprintf(stdout, "get pty device for exec %s\n", ptmx);
-
-	e->stdinev.fd = ptymaster;
-	e->stdoutev.fd = dup(ptymaster);
-	if (e->errseq == 0) {
-		e->stderrev.fd = dup(e->stdoutev.fd);
-	}
-	fprintf(stdout, "%s pts event %p, fd %d %d\n",
-		__func__, &e->stdinev, ptymaster, e->ptyfd);
+	e->ptyfd = ptymaster;
 	return 0;
 }
 
-static int hyper_dup_exec_tty(struct hyper_exec *e)
+static int hyper_install_process_stdio(struct hyper_exec *e, struct stdio_config *io)
 {
 	int ret = -1;
 
 	fprintf(stdout, "%s\n", __func__);
-	setsid();
 
 	if (e->tty) {
 		char ptmx[512];
+		int ptyslave;
+
 		sprintf(ptmx, "/dev/pts/%d", e->ptyno);
-		// reopen slave ptyfd for correcting the symlink path of the /dev/fd/1
-		e->ptyfd = open(ptmx, O_RDWR | O_CLOEXEC);
-		if (e->ptyfd < 0 || ioctl(e->ptyfd, TIOCSCTTY, NULL) < 0) {
+		ptyslave = open(ptmx, O_RDWR | O_CLOEXEC);
+		if (ptyslave < 0 || ioctl(ptyslave, TIOCSCTTY, NULL) < 0) {
 			perror("ioctl pty device for execcmd failed");
 			goto out;
 		}
-		e->stdinfd = e->ptyfd;
-		e->stdoutfd = e->ptyfd;
+		io->stdinfd = ptyslave;
+		io->stdoutfd = ptyslave;
 		if (e->errseq == 0)
-			e->stderrfd = e->ptyfd;
-		close(e->stdinev.fd);
-		close(e->stdoutev.fd);
-		close(e->stderrev.fd);
+			io->stderrfd = ptyslave;
 	}
 
-	fflush(stdout);
+	fflush(NULL);
 
-	if (dup2(e->stdinfd, STDIN_FILENO) < 0) {
+	if (dup2(io->stdinfd, STDIN_FILENO) < 0) {
 		perror("dup tty device to stdin failed");
 		goto out;
 	}
 
-	if (dup2(e->stdoutfd, STDOUT_FILENO) < 0) {
+	if (dup2(io->stdoutfd, STDOUT_FILENO) < 0) {
 		perror("dup tty device to stdout failed");
 		goto out;
 	}
 
-	if (dup2(e->stderrfd, STDERR_FILENO) < 0) {
+	if (dup2(io->stderrfd, STDERR_FILENO) < 0) {
 		perror("dup err pipe to stderr failed");
 		goto out;
 	}
 
+	/*
+	 * we are going to execvp(), all of the io->stdinfd, io->stdoutfd and
+	 * io->stderrfd are O_CLOEXEC, we don't need to close them explicitly
+	 */
 	ret = 0;
 out:
 	return ret;
 }
 
-static int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod)
+static int hyper_setup_stdio_events(struct hyper_exec *exec, struct stdio_config *io)
 {
-	fprintf(stdout, "hyper_init_event container pts event %p, ops %p, fd %d\n",
-		&exec->stdinev, &in_ops, exec->stdinev.fd);
+	if (exec->tty) {
+		io->stdinevfd = dup(exec->ptyfd);
+		io->stdoutevfd = dup(exec->ptyfd);
+		if (exec->errseq == 0) {
+			io->stderrevfd = dup(exec->ptyfd);
+		}
+	}
 
-	if (exec->seq == 0)
-		return 0;
-
-	if (hyper_init_event(&exec->stdinev, &in_ops, pod) < 0 ||
+	fprintf(stdout, "hyper_init_event exec stdin event %p, ops %p, fd %d\n",
+		&exec->stdinev, &in_ops, io->stdinevfd);
+	exec->stdinev.fd = io->stdinevfd;
+	if (hyper_init_event(&exec->stdinev, &in_ops, NULL) < 0 ||
 	    hyper_add_event(ctl.efd, &exec->stdinev, EPOLLOUT) < 0) {
 		fprintf(stderr, "add container stdin event failed\n");
 		return -1;
 	}
 	exec->ref++;
 
-	if (hyper_init_event(&exec->stdoutev, &out_ops, pod) < 0 ||
+	fprintf(stdout, "hyper_init_event exec stdout event %p, ops %p, fd %d\n",
+		&exec->stdoutev, &out_ops, io->stdoutevfd);
+	exec->stdoutev.fd = io->stdoutevfd;
+	if (hyper_init_event(&exec->stdoutev, &out_ops, NULL) < 0 ||
 	    hyper_add_event(ctl.efd, &exec->stdoutev, EPOLLIN) < 0) {
 		fprintf(stderr, "add container stdout event failed\n");
 		return -1;
 	}
 	exec->ref++;
 
-	if (hyper_init_event(&exec->stderrev, &err_ops, pod) < 0 ||
+	fprintf(stdout, "hyper_init_event exec stderr event %p, ops %p, fd %d\n",
+		&exec->stderrev, &err_ops, io->stderrevfd);
+	exec->stderrev.fd = io->stderrevfd;
+	if (hyper_init_event(&exec->stderrev, &err_ops, NULL) < 0 ||
 	    hyper_add_event(ctl.efd, &exec->stderrev, EPOLLIN) < 0) {
 		fprintf(stderr, "add container stderr event failed\n");
 		return -1;
@@ -490,19 +467,19 @@ static int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod)
 	return 0;
 }
 
-static int hyper_do_exec_cmd(struct hyper_exec *exec, struct hyper_pod *pod, int pipe)
+static int hyper_do_exec_cmd(struct hyper_exec *exec, int pipe, struct stdio_config *io)
 {
 	struct hyper_container *c;
 
-	if (hyper_enter_sandbox(pod, pipe) < 0) {
+	if (hyper_enter_sandbox(exec->pod, pipe) < 0) {
 		perror("enter pidns of pod init failed");
 		hyper_send_type(pipe, -1);
 		goto out;
 	}
 
-	c = hyper_find_container(pod, exec->id);
+	c = hyper_find_container(exec->pod, exec->container_id);
 	if (c == NULL) {
-		fprintf(stderr, "can not find container %s\n", exec->id);
+		fprintf(stderr, "can not find container %s\n", exec->container_id);
 		goto out;
 	}
 
@@ -510,7 +487,10 @@ static int hyper_do_exec_cmd(struct hyper_exec *exec, struct hyper_pod *pod, int
 		perror("fail to enter container ns");
 		goto out;
 	}
-	chdir("/");
+	if (chdir("/") < 0) {
+		perror("fail to change to the root of the rootfs");
+		goto out;
+	}
 
 	/* TODO: merge container env to exec env in hyperd */
 	if (hyper_setup_env(c->exec.envs, c->exec.envs_num) < 0) {
@@ -520,20 +500,20 @@ static int hyper_do_exec_cmd(struct hyper_exec *exec, struct hyper_pod *pod, int
 
 	// set early env. the container env config can overwrite it
 	setenv("HOME", "/root", 1);
-	setenv("HOSTNAME", pod->hostname, 1);
+	setenv("HOSTNAME", exec->pod->hostname, 1);
 	if (exec->tty)
 		setenv("TERM", "xterm", 1);
 	else
 		unsetenv("TERM");
 
-	hyper_exec_process(exec);
+	hyper_exec_process(exec, io);
 
 out:
 	_exit(125);
 }
 
 // do the exec, no return
-static void hyper_exec_process(struct hyper_exec *exec)
+static void hyper_exec_process(struct hyper_exec *exec, struct stdio_config *io)
 {
 	if (sigprocmask(SIG_SETMASK, &orig_mask, NULL) < 0) {
 		perror("sigprocmask restore mask failed");
@@ -556,24 +536,27 @@ static void hyper_exec_process(struct hyper_exec *exec)
 		goto exit;
 	}
 
-	if (hyper_dup_exec_tty(exec) < 0) {
+	setsid();
+
+	if (hyper_install_process_stdio(exec, io) < 0) {
 		fprintf(stderr, "dup pts to exec stdio failed\n");
 		goto exit;
 	}
 
 	if (execvp(exec->argv[0], exec->argv) < 0) {
+		// perror possibly changes the errno.
+		int err = errno;
 		perror("exec failed");
-
 		 /* the exit codes follow the `chroot` standard,
 		    see docker/docs/reference/run.md#exit-status */
-		if (errno == ENOENT)
+		if (err == ENOENT)
 			_exit(127);
-		else if (errno == EACCES)
+		else if (err == EACCES)
 			_exit(126);
 	}
 
 exit:
-	fflush(stdout);
+	fflush(NULL);
 	_exit(125);
 }
 
@@ -581,7 +564,7 @@ static void hyper_free_exec(struct hyper_exec *exec)
 {
 	int i;
 
-	free(exec->id);
+	free(exec->container_id);
 
 	for (i = 0; i < exec->argc; i++) {
 		//fprintf(stdout, "argv %d %s\n", i, exec->argv[i]);
@@ -604,6 +587,7 @@ int hyper_exec_cmd(char *json, int length)
 		return -1;
 	}
 
+	exec->pod = &global_pod;
 	int ret = hyper_run_process(exec);
 	if (ret < 0) {
 		hyper_free_exec(exec);
@@ -613,30 +597,22 @@ int hyper_exec_cmd(char *json, int length)
 
 int hyper_run_process(struct hyper_exec *exec)
 {
-	struct hyper_pod *pod = &global_pod;
 	int pipe[2] = {-1, -1};
 	int pid, ret = -1;
 	uint32_t type;
+	struct stdio_config io = {-1, -1,-1, -1,-1, -1};
 
-	if (exec->argv == NULL) {
+	if (exec->argv == NULL || exec->seq == 0 || exec->container_id == NULL || strlen(exec->container_id) == 0) {
 		fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n",
-			exec->argv, exec->seq, exec->id);
+			exec->argv, exec->seq, exec->container_id);
 		goto out;
 	}
 
-	if (hyper_setup_exec_tty(exec) < 0) {
+	if (hyper_setup_stdio(exec, &io) < 0) {
 		fprintf(stderr, "setup exec tty failed\n");
 		goto out;
 	}
 
-	if (hyper_watch_exec_pty(exec, pod) < 0) {
-		fprintf(stderr, "add pts master event failed\n");
-		goto close_tty;
-	}
-
-	list_add_tail(&exec->list, &pod->exec_head);
-	exec->ref++;
-
 	if (pipe2(pipe, O_CLOEXEC) < 0) {
 		perror("create pipe between pod init execcmd failed");
 		goto close_tty;
@@ -644,22 +620,32 @@ int hyper_run_process(struct hyper_exec *exec)
 
 	pid = fork();
 	if (pid < 0) {
-		perror("clone hyper_do_exec_cmd failed");
+		perror("fork prerequisite process failed");
 		goto close_tty;
 	} else if (pid == 0) {
-		hyper_do_exec_cmd(exec, pod, pipe[1]);
+		hyper_do_exec_cmd(exec, pipe[1], &io);
 	}
-	fprintf(stdout, "do_exec_cmd pid %d\n", pid);
+	fprintf(stdout, "prerequisite process pid %d\n", pid);
 
 	if (hyper_get_type(pipe[0], &type) < 0 || (int)type < 0) {
-		fprintf(stderr, "hyper init doesn't get execcmd ready message\n");
+		fprintf(stderr, "run process failed\n");
 		goto close_tty;
 	}
-	exec->pid = type;
 
-	fprintf(stdout, "%s get ready message %"PRIu32 "\n", __func__, type);
+	if (hyper_setup_stdio_events(exec, &io) < 0) {
+		fprintf(stderr, "add pts master event failed\n");
+		goto close_tty;
+	}
+
+	exec->pid = type;
+	list_add_tail(&exec->list, &exec->pod->exec_head);
+	exec->ref++;
+	fprintf(stdout, "%s process pid %d\n", __func__, exec->pid);
 	ret = 0;
 out:
+	close(io.stdinfd);
+	close(io.stdoutfd);
+	close(io.stderrfd);
 	close(pipe[0]);
 	close(pipe[1]);
 	return ret;
@@ -670,9 +656,9 @@ close_tty:
 	list_del_init(&exec->list);
 
 	close(exec->ptyfd);
-	close(exec->stdinfd);
-	close(exec->stdoutfd);
-	close(exec->stderrfd);
+	close(io.stdinevfd);
+	close(io.stdoutevfd);
+	close(io.stderrevfd);
 	goto out;
 }
 
@@ -699,8 +685,7 @@ out:
 	return ret;
 }
 
-static int hyper_release_exec(struct hyper_exec *exec,
-			      struct hyper_pod *pod)
+static int hyper_release_exec(struct hyper_exec *exec)
 {
 	if (--exec->ref != 0) {
 		fprintf(stdout, "still have %d user of exec\n", exec->ref);
@@ -716,31 +701,28 @@ static int hyper_release_exec(struct hyper_exec *exec,
 
 	list_del_init(&exec->list);
 
-	hyper_send_exec_eof(exec, 0);
+	hyper_send_exec_eof(exec);
 
-	hyper_send_exec_code(exec, 0);
+	hyper_send_exec_code(exec);
 
 	fprintf(stdout, "%s exit code %" PRIu8"\n", __func__, exec->code);
 	if (exec->init) {
-		fprintf(stdout, "%s container init exited, type %d, remains %d, policy %d\n",
-			__func__, pod->type, pod->remains, pod->policy);
+		fprintf(stdout, "%s container init exited %s, remains %d\n",
+			__func__, exec->pod->req_destroy?"manually":"automatically", exec->pod->remains);
 
 		// TODO send finish of this container and full cleanup
-		if (--pod->remains > 0)
+		if (--exec->pod->remains > 0)
 			return 0;
 
-		if (pod->type == STOPPOD) {
-			/* stop pod manually, hyper doesn't care the pod finished codes */
-			hyper_send_msg_block(ctl.chan.fd, ACK, 0, NULL);
-		} else if (pod->type == DESTROYPOD) {
+		if (exec->pod->req_destroy) {
 			/* shutdown vm manually, hyper doesn't care the pod finished codes */
-			hyper_shutdown(0);
+			hyper_pod_destroyed(0);
 		} else {
 			/* send out pod finish message, hyper will decide if restart pod or not */
-			hyper_send_pod_finished(pod);
+			hyper_send_pod_finished(exec->pod);
 		}
 
-		hyper_cleanup_pod(pod);
+		hyper_cleanup_pod(exec->pod);
 		return 0;
 	}
 
@@ -843,36 +825,18 @@ int hyper_handle_exec_exit(struct hyper_pod *pod, int pid, uint8_t code)
 	}
 
 	fprintf(stdout, "%s exec exit pid %d, seq %" PRIu64 ", container %s\n",
-		__func__, exec->pid, exec->seq, exec->id);
+		__func__, exec->pid, exec->seq, exec->container_id);
 
 	exec->code = code;
 	exec->exit = 1;
 
 	close(exec->ptyfd);
 	exec->ptyfd = -1;
-	close(exec->stdinfd);
-	exec->stdinfd = -1;
-	close(exec->stdoutfd);
-	exec->stdoutfd = -1;
-	close(exec->stderrfd);
-	exec->stderrfd = -1;
-
-	hyper_release_exec(exec, pod);
 
 	if (exec->init)
 		hyper_kill_container_processes(container_of(exec, struct hyper_container, exec));
 
+	hyper_release_exec(exec);
+
 	return 0;
 }
-
-void hyper_cleanup_exec(struct hyper_pod *pod)
-{
-	struct hyper_exec *exec, *next;
-
-	list_for_each_entry_safe(exec, next, &pod->exec_head, list) {
-		fprintf(stdout, "send eof for exec seq %" PRIu64 "\n", exec->seq);
-		if (hyper_send_exec_eof(exec, 1) < 0 ||
-		    hyper_send_exec_code(exec, 1) < 0)
-			fprintf(stderr, "send eof failed\n");
-	}
-}

src/exec.h

@@ -11,6 +11,8 @@ struct env {
 
 struct hyper_exec {
 	struct list_head	list;
+	struct hyper_pod	*pod;
+
 	struct hyper_event	stdinev;
 	struct hyper_event	stdoutev;
 	struct hyper_event	stderrev;
@@ -18,16 +20,13 @@ struct hyper_exec {
 	int			ptyno;
 	int			init;
 	int			ptyfd;
-	int			stdinfd;
-	int			stdoutfd;
-	int			stderrfd;
 	uint8_t			close_stdin_request;
 	uint8_t			code;
 	uint8_t			exit;
 	uint8_t			ref;
 
 	// configs
-	char			*id;
+	char			*container_id;
 	char			*user;
 	char			*group;
 	char			**additional_groups;
@@ -49,6 +48,5 @@ int hyper_run_process(struct hyper_exec *e);
 struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid);
 struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq);
 int hyper_handle_exec_exit(struct hyper_pod *pod, int pid, uint8_t code);
-void hyper_cleanup_exec(struct hyper_pod *pod);
 
 #endif

src/hyper.h

@@ -7,6 +7,7 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#include "api.h"
 #include "net.h"
 #include "list.h"
 #include "exec.h"
@@ -14,37 +15,8 @@
 #include "container.h"
 #include "portmapping.h"
 
-enum {
-	RESERVED,
-	STARTPOD,
-	GETPOD,
-	STOPPOD,
-	DESTROYPOD,
-	RESTARTCONTAINER,
-	EXECCMD,
-	CMDFINISHED,
-	READY,
-	ACK,
-	ERROR,
-	WINSIZE,
-	PING,
-	PODFINISHED,
-	NEXT,
-	WRITEFILE,
-	READFILE,
-	NEWCONTAINER,
-	KILLCONTAINER,
-	ONLINECPUMEM,
-	SETUPINTERFACE,
-	SETUPROUTE,
-	REMOVECONTAINER,
-};
-
-enum {
-	POLICY_NEVER,
-	POLICY_ALWAYS,
-	POLICY_ONFAILURE,
-};
+/* Path to rootfs shared directory */
+#define SHARED_DIR "/tmp/hyper/shared"
 
 struct hyper_pod {
 	struct hyper_interface	*iface;
@@ -59,10 +31,9 @@ struct hyper_pod {
 	uint32_t		i_num;
 	uint32_t		r_num;
 	uint32_t		d_num;
-	uint32_t		type;
 	/* how many containers are running */
 	uint32_t		remains;
-	uint8_t			policy;
+	int			req_destroy;
 	int			efd;
 };
 
@@ -79,18 +50,11 @@ struct hyper_win_size {
 	uint64_t	seq;
 };
 
-struct hyper_reader {
+struct file_command {
 	char		*id;
 	char		*file;
 };
 
-struct hyper_writter {
-	char		*id;
-	char		*file;
-	uint8_t		*data;
-	int		len;
-};
-
 struct hyper_ctl {
 	int			efd;
 	struct hyper_event	tty;
@@ -120,6 +84,7 @@ static inline int hyper_create(char *hyper_path)
 int hyper_open_serial(char *tty);
 void hyper_cleanup_pod(struct hyper_pod *pod);
 int hyper_enter_sandbox(struct hyper_pod *pod, int pidpipe);
+void hyper_pod_destroyed(int failed);
 
 extern struct hyper_pod global_pod;
 extern struct hyper_ctl ctl;

src/init.c

@@ -41,34 +41,38 @@ struct hyper_ctl ctl;
 sigset_t orig_mask;
 
 static int hyper_handle_exit(struct hyper_pod *pod);
-static int hyper_stop_pod(struct hyper_pod *pod);
 
 static int hyper_set_win_size(char *json, int length)
 {
-	struct hyper_win_size ws;
 	struct winsize size;
 	struct hyper_exec *exec;
 	int ret;
 
 	fprintf(stdout, "call hyper_win_size, json %s, len %d\n", json, length);
-	if (hyper_parse_winsize(&ws, json, length) < 0) {
+	JSON_Value *value = hyper_json_parse(json, length);
+	if (value == NULL) {
 		fprintf(stderr, "set term size failed\n");
-		return -1;
+		ret = -1;
+		goto out;
 	}
+	const uint64_t seq = (uint64_t)json_object_get_number(json_object(value), "seq");
 
-	exec = hyper_find_exec_by_seq(&global_pod, ws.seq);
+	exec = hyper_find_exec_by_seq(&global_pod, seq);
 	if (exec == NULL) {
-		fprintf(stdout, "can not find exec whose seq is %" PRIu64"\n", ws.seq);
-		return 0;
+		fprintf(stdout, "can not find exec whose seq is %" PRIu64"\n", seq);
+		ret = 0;
+		goto out;
 	}
 
-	size.ws_row = ws.row;
-	size.ws_col = ws.column;
+	size.ws_row = (int)json_object_get_number(json_object(value), "row");
+	size.ws_col = (int)json_object_get_number(json_object(value), "column");
 
 	ret = ioctl(exec->ptyfd, TIOCSWINSZ, &size);
 	if (ret < 0)
 		perror("cannot ioctl to set pty device term size");
 
+out:
+	json_value_free(value);
 	return ret;
 }
 
@@ -235,7 +239,7 @@ static int hyper_pod_init(void *data)
 	}
 
 	if (hyper_send_type(arg->ctl_pipe[1], READY) < 0) {
-		fprintf(stderr, "container init send ready message failed\n");
+		fprintf(stderr, "pod init send ready message failed\n");
 		goto fail;
 	}
 
@@ -283,7 +287,7 @@ static int hyper_setup_pod_init(struct hyper_pod *pod)
 
 	uint32_t type;
 	void *stack;
-	int ret = -1;
+	int ret = -1, init_pid;
 
 	if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) {
 		perror("create pipe between hyper init and pod init failed");
@@ -292,23 +296,23 @@ static int hyper_setup_pod_init(struct hyper_pod *pod)
 
 	stack = malloc(stacksize);
 	if (stack == NULL) {
-		perror("fail to allocate stack for container init");
+		perror("fail to allocate stack for pod init");
 		goto out;
 	}
 
 	arg.pod = pod;
 
-	pod->init_pid = clone(hyper_pod_init, stack + stacksize, flags, &arg);
+	init_pid = clone(hyper_pod_init, stack + stacksize, flags, &arg);
 	free(stack);
-	if (pod->init_pid < 0) {
-		perror("create container init process failed");
+	if (init_pid < 0) {
+		perror("create pod init process failed");
 		goto out;
 	}
-	fprintf(stdout, "pod init pid %d\n", pod->init_pid);
+	fprintf(stdout, "pod init pid %d\n", init_pid);
 
-	/* Wait for container start */
+	/* Wait for pod init start */
 	if (hyper_get_type(arg.ctl_pipe[0], &type) < 0) {
-		perror("get container init ready message failed");
+		perror("get pod init ready message failed");
 		goto out;
 	}
 
@@ -317,13 +321,11 @@ static int hyper_setup_pod_init(struct hyper_pod *pod)
 		goto out;
 	}
 
+	pod->init_pid = init_pid;
 	ret = 0;
 out:
 	close(arg.ctl_pipe[1]);
 	close(arg.ctl_pipe[0]);
-	if (ret < 0) {
-		hyper_stop_pod(pod);
-	}
 	return ret;
 }
 
@@ -416,12 +418,12 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 	struct vbsf_mount_info_new mntinf;
 
 	if (pod->share_tag == NULL) {
-		fprintf(stdout, "no shared directroy\n");
+		fprintf(stdout, "no shared directory\n");
 		return 0;
 	}
 
-	if (hyper_mkdir("/tmp/hyper/shared", 0755) < 0) {
-		perror("fail to create /tmp/hyper/shared");
+	if (hyper_mkdir(SHARED_DIR, 0755) < 0) {
+		perror("fail to create " SHARED_DIR);
 		return -1;
 	}
 
@@ -435,7 +437,7 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 	mntinf.fmode		= ~0U;
 	strcpy(mntinf.name, pod->share_tag);
 
-	if (mount(NULL, "/tmp/hyper/shared", "vboxsf",
+	if (mount(NULL, SHARED_DIR, "vboxsf",
 		  MS_NODEV, &mntinf) < 0) {
 		perror("fail to mount shared dir");
 		return -1;
@@ -447,16 +449,16 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 static int hyper_setup_shared(struct hyper_pod *pod)
 {
 	if (pod->share_tag == NULL) {
-		fprintf(stdout, "no shared directroy\n");
+		fprintf(stdout, "no shared directory\n");
 		return 0;
 	}
 
-	if (hyper_mkdir("/tmp/hyper/shared", 0755) < 0) {
-		perror("fail to create /tmp/hyper/shared");
+	if (hyper_mkdir(SHARED_DIR, 0755) < 0) {
+		perror("fail to create " SHARED_DIR);
 		return -1;
 	}
 
-	if (mount(pod->share_tag, "/tmp/hyper/shared", "9p",
+	if (mount(pod->share_tag, SHARED_DIR, "9p",
 		  MS_MGC_VAL| MS_NODEV, "trans=virtio") < 0) {
 
 		perror("fail to mount shared dir");
@@ -469,7 +471,7 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 
 static int hyper_setup_pod(struct hyper_pod *pod)
 {
-	/* create tmp proc directroy */
+	/* create tmp proc directory */
 	if (hyper_mkdir("/tmp/hyper/proc", 0755) < 0) {
 		perror("create tmp proc failed");
 		return -1;
@@ -517,11 +519,17 @@ static void hyper_print_uptime(void)
 	close(fd);
 }
 
+void hyper_pod_destroyed(int failed)
+{
+	hyper_send_msg_block(ctl.chan.fd, failed?ERROR:ACK, 0, NULL);
+	hyper_shutdown();
+}
+
 static int hyper_destroy_pod(struct hyper_pod *pod, int error)
 {
-	if (pod->init_pid == 0) {
+	if (pod->init_pid == 0 || pod->remains == 0) {
 		/* Pod stopped, just shutdown */
-		hyper_shutdown(error);
+		hyper_pod_destroyed(error);
 	} else {
 		/* Kill pod */
 		hyper_term_all(pod);
@@ -581,11 +589,13 @@ static int hyper_new_container(char *json, int length)
 	ret = hyper_setup_container(c, pod);
 	if (ret >= 0)
 		ret = hyper_run_process(&c->exec);
+
 	if (ret < 0) {
 		//TODO full grace cleanup
 		hyper_cleanup_container(c, pod);
+	} else {
+		pod->remains++;
 	}
-	pod->remains++;
 
 	return ret;
 }
@@ -646,185 +656,81 @@ out:
 	return ret;
 }
 
-static int hyper_cmd_write_file(char *json, int length)
-{
-	struct hyper_writter writter;
-	struct hyper_container *c;
-	struct hyper_pod *pod = &global_pod;
-	int pipe[2] = {-1, -1};
-	int pid, mntns = -1, fd;
-	int len = 0, size, ret = -1;
-
-	fprintf(stdout, "%s\n", __func__);
-
-	if (hyper_parse_write_file(&writter, json, length) < 0) {
-		goto out;
-	}
-
-	c = hyper_find_container(pod, writter.id);
-	if (c == NULL) {
-		fprintf(stderr, "can not find container whose id is %s\n", writter.id);
-		goto out;
-	}
-
-	if (pipe2(pipe, O_CLOEXEC) < 0) {
-		perror("create writter pipe failed");
-		goto out;
-	}
-
-	mntns = c->ns;
-	if (mntns < 0) {
-		perror("fail to open mnt ns");
-		goto out;
-	}
-
-	pid = fork();
-	if (pid < 0) {
-		perror("fail to fork writter process");
-		goto out;
-	} else if (pid > 0) {
-		uint32_t type;
-
-		if (hyper_get_type(pipe[0], &type) < 0 || type != READY) {
-			fprintf(stderr, "get incorrect message type %d, expect READY\n", type);
-			goto out;
-		}
-
-		ret = 0;
-		goto out;
-	}
-
-	/* TODO: wait for container finishing setup root */
-	if (setns(mntns, CLONE_NEWNS) < 0) {
-		perror("fail to enter container ns");
-		goto exit;
-	}
-
-	fprintf(stdout, "write file %s, data len %d\n", writter.file, writter.len);
-
-	fd = open(writter.file, O_CREAT| O_TRUNC| O_WRONLY, 0644);
-	if (fd < 0) {
-		perror("fail to open target file");
-		goto exit;
-	}
-
-	while(len < writter.len) {
-		size = write(fd, writter.data + len, writter.len - len);
-
-		if (size < 0) {
-			if (errno == EINTR)
-				continue;
-
-			perror("fail to write data to file");
-			goto exit;
-		}
-
-		len += size;
-	}
-	ret = 0;
-exit:
-	hyper_send_type(pipe[1], ret ? ERROR : READY);
-	_exit(0);
-out:
-	close(pipe[0]);
-	close(pipe[1]);
-	free(writter.id);
-	free(writter.file);
-	free(writter.data);
-
-	return 0;
-}
-
 struct hyper_file_arg {
-	int		mntns;
-	int		pipe[2];
-	char		*file;
-	char		root[128];
-	uint32_t	*datalen;
-	uint8_t		**data;
+	int 		rw;
+	int 		mntns;
+	int 		pipe[2];
+	char 		*file;
 };
 
-static int hyper_do_cmd_read_file(void *data)
+static int hyper_open_container_file(void *data)
 {
-	struct stat st;
-	int len = 0, size, fd, ret = -1;
 	struct hyper_file_arg *arg = data;
+	int fd = -1, ret = -1, size;
 
-	/* TODO: wait for container finishing setup root */
 	if (setns(arg->mntns, CLONE_NEWNS) < 0) {
 		perror("fail to enter container ns");
-		goto err;
+		goto exit;
 	}
 
-	fprintf(stdout, "read file %s\n", arg->file);
-
-	if (stat(arg->file, &st) < 0) {
-		perror("fail to state file");
-		goto err;
+	if (arg->rw == WRITEFILE) {
+		fd = open(arg->file, O_CREAT| O_TRUNC| O_WRONLY, 0644);
+	} else {
+		fd = open(arg->file, O_RDONLY);
 	}
-
-	*arg->datalen = st.st_size;
-	*arg->data = malloc(*arg->datalen);
-	if (*arg->data == NULL) {
-		fprintf(stderr, "allocate memory for reading file failed\n");
-		goto err;
-	}
-
-	fd = open(arg->file, O_RDONLY);
 	if (fd < 0) {
 		perror("fail to open target file");
-		goto err;
+		goto exit;
 	}
-
-	fprintf(stdout, "file length %d\n", *arg->datalen);
-	while(len < *arg->datalen) {
-		size = read(fd, *arg->data + len, *arg->datalen - len);
-
-		if (size < 0) {
-			if (errno == EINTR)
-				continue;
-
-			perror("fail to read data from file");
-			goto err;
-		}
-
-		len += size;
-	}
-
-	fprintf(stdout, "read data %s\n", *arg->data);
 	ret = 0;
-err:
-	hyper_send_type(arg->pipe[1], ret ? ERROR : READY);
-	_exit(ret);
+
+exit:
+	size = write(arg->pipe[1], &fd, sizeof(fd));
+	if (size != sizeof(fd) && ret == 0) {
+		ret = -1;
+	}
+	exit(ret);
 }
 
-static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_t **data)
+static int hyper_cmd_rw_file(char *json, int length, uint32_t *rdatalen, uint8_t **rdata, int rw)
 {
-	struct hyper_reader reader;
-	struct hyper_container *c;
-	struct hyper_pod *pod = &global_pod;
+	struct file_command cmd = {
+		.id = NULL,
+		.file = NULL,
+	};
 	struct hyper_file_arg arg = {
 		.pipe = {-1, -1},
+		.rw = rw,
 	};
-	int stacksize = getpagesize() * 4;
+	struct hyper_container *c;
+	struct hyper_pod *pod = &global_pod;
+	char *data = NULL;
 	void *stack = NULL;
-	int pid, ret = -1, status;
-	uint32_t type;
+	int stacksize = getpagesize() * 4;
+	int fd = -1, len = 0, ret = -1, datalen = 0;
+	int pid, size;
 
-	fprintf(stdout, "%s\n", __func__);
+	fprintf(stdout, "%s: %s\n", __func__, rw == WRITEFILE ? "write" : "read");
 
-	if (hyper_parse_read_file(&reader, json, length) < 0) {
-		goto out;
+	if (rw == WRITEFILE) {
+		// TODO: send the data via hyperstream rather than append it at the end of the command	
+		data = strchr(json, '}');
+		if (data == NULL) {
+			goto out;
+		}
+		data++;
+		datalen = length - (data - json);
+		length = data - json;
 	}
 
-	c = hyper_find_container(pod, reader.id);
+	if (hyper_parse_file_command(&cmd, json, length) < 0) {
+		goto out;
+	}
+	arg.file = cmd.file;
+
+	c = hyper_find_container(pod, cmd.id);
 	if (c == NULL) {
-		fprintf(stderr, "can not find container whose id is %s\n", reader.id);
-		goto out;
-	}
-
-	if (pipe2(arg.pipe, O_CLOEXEC) < 0) {
-		perror("create reader pipe failed");
+		fprintf(stderr, "can not find container whose id is %s\n", cmd.id);
 		goto out;
 	}
 
@@ -834,9 +740,10 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_
 		goto out;
 	}
 
-	arg.file = reader.file;
-	arg.datalen = datalen;
-	arg.data = data;
+	if (pipe2(arg.pipe, O_CLOEXEC) < 0) {
+		perror("create pipe failed");
+		goto out;
+	}
 
 	stack = malloc(stacksize);
 	if (stack == NULL) {
@@ -844,28 +751,63 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_
 		goto out;
 	}
 
-	pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM| SIGQUIT, &arg);
+	pid = clone(hyper_open_container_file, stack + stacksize, CLONE_FILES | SIGCHLD, &arg);
 	if (pid < 0) {
-		perror("fail to fork writter process");
+		perror("fail to fork child process");
 		goto out;
 	}
 
-	if (waitpid(pid, &status, __WCLONE) <= 0) {
-		perror("waiting hyper_do_cmd_read_file finish failed");
+	size = read(arg.pipe[0], &fd, sizeof(fd));
+	if (size != sizeof(fd)) {
+		perror("fail to read fd from pipe");
+		goto out;
+	}
+	if (fd < 0) {
+		perror("child open target file failed");
 		goto out;
 	}
 
-	if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) {
-		fprintf(stderr, "%s to incorrect type %" PRIu32 "\n", __func__, type);
-		goto out;
+	if (rw == READFILE) {
+		struct stat st;
+		if(fstat(fd, &st) < 0) {
+			perror("fail to state file");
+			goto out;
+		}
+		*rdatalen = datalen = st.st_size;
+		data = malloc(datalen);
+		*rdata = (uint8_t *) data;
+		if (*rdata == NULL) {
+			fprintf(stderr, "allocate memory for reading file failed\n");
+			goto out;
+		}
+		fprintf(stdout, "file length %d\n", *rdatalen);
 	}
 
+	while(len < datalen) {
+		if (rw == WRITEFILE) {
+			size = write(fd, data + len, datalen - len);
+		} else {
+			size = read(fd, data + len, datalen - len);
+		}
+
+		if (size < 0) {
+			if (errno == EINTR)
+				continue;
+
+			perror("fail to operate data to file");
+			goto out;
+		}
+
+		len += size;
+	}
 	ret = 0;
+
 out:
+	close(fd);
 	close(arg.pipe[0]);
 	close(arg.pipe[1]);
-	free(reader.id);
-	free(reader.file);
+	free(cmd.id);
+	free(cmd.file);
 	free(stack);
 
 	return ret;
@@ -892,20 +834,20 @@ static void hyper_cleanup_hostname(struct hyper_pod *pod)
 static void hyper_cleanup_shared(struct hyper_pod *pod)
 {
 	if (pod->share_tag == NULL) {
-		fprintf(stdout, "no shared directroy\n");
+		fprintf(stdout, "no shared directory\n");
 		return;
 	}
 
 	free(pod->share_tag);
 	pod->share_tag = NULL;
-	if (umount("/tmp/hyper/shared") < 0 &&
-	    umount2("/tmp/hyper/shared", MNT_DETACH)) {
+	if (umount(SHARED_DIR) < 0 &&
+	    umount2(SHARED_DIR, MNT_DETACH)) {
 		perror("fail to umount shared dir");
 		return;
 	}
 
-	if (rmdir("/tmp/hyper/shared") < 0)
-		perror("fail to delete /tmp/hyper/shared");
+	if (rmdir(SHARED_DIR) < 0)
+		perror("fail to delete " SHARED_DIR);
 
 	sync();
 }
@@ -924,20 +866,6 @@ void hyper_cleanup_pod(struct hyper_pod *pod)
 	hyper_cleanup_hostname(pod);
 }
 
-static int hyper_stop_pod(struct hyper_pod *pod)
-{
-	fprintf(stdout, "hyper_stop_pod init_pid %d\n", pod->init_pid);
-	if (pod->init_pid == 0) {
-		fprintf(stdout, "container init pid is already exit\n");
-		hyper_send_type(ctl.chan.fd, ACK);
-		return 0;
-	}
-
-	pod->init_pid = 0;
-	hyper_term_all(pod);
-	return 0;
-}
-
 static int hyper_setup_ctl_channel(char *name)
 {
 	int ret = hyper_open_channel(name, 0);
@@ -977,7 +905,7 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len)
 
 	seq = hyper_get_be64(rbuf->data);
 
-	dprintf(stdout, "\n%s seq %" PRIu64", len %" PRIu32"\n", __func__, seq, len - 12);
+	fprintf(stdout, "\n%s seq %" PRIu64", len %" PRIu32"\n", __func__, seq, len - 12);
 
 	exec = hyper_find_exec_by_seq(pod, seq);
 	if (exec == NULL) {
@@ -1000,28 +928,20 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len)
 		return 0;
 	}
 
-	dprintf(stdout, "find exec %s pid %d, seq is %" PRIu64 "\n",
-		exec->id ? exec->id : "pod", exec->pid, exec->seq);
-	// if exec is exited, the event fd of exec is invalid. don't accept any input.
-	if (exec->exit || exec->close_stdin_request) {
+	fprintf(stdout, "find exec %s pid %d, seq is %" PRIu64 "\n",
+		exec->container_id ? exec->container_id : "pod", exec->pid, exec->seq);
+	// if exec is exited or stdin is closed by process, the event fd of exec is invalid.
+	// don't accept any input.
+	if (exec->exit || exec->close_stdin_request || exec->stdinev.fd < 0) {
 		fprintf(stdout, "exec seq %" PRIu64 " exited, don't accept any input\n", exec->seq);
 		return 0;
 	}
 
-	wbuf = &exec->stdinev.wbuf;
-
-	size = wbuf->size - wbuf->get;
-	if (size == 0)
-		return 0;
-
-	/* Data may lost since pts buffer is full. do not allow one exec pts occupy all
-	 * of the tty buff. */
-	if (size > (len - 12))
-		size = (len - 12);
-
+	size = len - STREAM_HEADER_SIZE;
 	/* size == 0 means we had received eof */
 	if (size == 0 && !exec->tty) {
 		exec->close_stdin_request = 1;
+		fprintf(stdout, "get close stdin request\n");
 		/* we can't hup the stdinev here, force hup on next write */
 		if (hyper_modify_event(ctl.efd, &exec->stdinev, EPOLLOUT) < 0) {
 			fprintf(stderr, "modify exec pts event to in & out failed\n");
@@ -1029,6 +949,12 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len)
 		}
 	}
 
+	wbuf = &exec->stdinev.wbuf;
+	if (size > (wbuf->size - wbuf->get)) {
+		/* buffer is full, discard the data */
+		/* TODO: properly handle the discard data */
+		size = wbuf->size - wbuf->get;
+	}
 	if (size > 0) {
 		memcpy(wbuf->data + wbuf->get, rbuf->data + 12, size);
 		wbuf->get += size;
@@ -1041,6 +967,47 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len)
 	return 0;
 }
 
+static int hyper_ttyfd_read(struct hyper_event *he, int efd)
+{
+	struct hyper_buf *buf = &he->rbuf;
+	uint32_t len;
+	int size;
+	int ret;
+
+	if (buf->get < STREAM_HEADER_SIZE) {
+		size = nonblock_read(he->fd, buf->data + buf->get, STREAM_HEADER_SIZE - buf->get);
+		if (size < 0) {
+			return size;
+		}
+		buf->get += size;
+		if (buf->get < STREAM_HEADER_SIZE) {
+			return 0;
+		}
+	}
+
+	len = hyper_get_be32(buf->data + STREAM_HEADER_LENGTH_OFFSET);
+	fprintf(stdout, "%s: get length %" PRIu32"\n", __func__, len);
+	if (len > buf->size) {
+		fprintf(stderr, "get length %" PRIu32", too long\n", len);
+		return -1;
+	}
+
+	size = nonblock_read(he->fd, buf->data + buf->get, len - buf->get);
+	if (size < 0) {
+		return size;
+	}
+	buf->get += size;
+	if (buf->get < len) {
+		return 0;
+	}
+
+	/* get and consume the whole data */
+	ret = hyper_ttyfd_handle(he, len);
+	buf->get = 0;
+
+	return ret == 0 ? 0 : -1;
+}
+
 static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 {
 	struct hyper_buf *buf = &de->rbuf;
@@ -1049,6 +1016,8 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 	uint8_t *data = NULL;
 	int i, ret = 0;
 
+	// append a null byte to it. hyper_channel_read() left this room for us.
+	buf->data[buf->get] = 0;
 	for (i = 0; i < buf->get; i++)
 		fprintf(stdout, "%0x ", buf->data[i]);
 
@@ -1057,17 +1026,22 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 	fprintf(stdout, "\n %s, type %" PRIu32 ", len %" PRIu32 "\n",
 		__func__, type, len);
 
-	pod->type = type;
 	switch (type) {
+	case GETVERSION:
+		data = malloc(4);
+		datalen = 4;
+		hyper_set_be32(data, APIVERSION);
+		break;
 	case STARTPOD:
 		ret = hyper_start_pod((char *)buf->data + 8, len - 8);
 		hyper_print_uptime();
 		break;
-	case STOPPOD:
-		hyper_stop_pod(pod);
-		return 0;
-		//break;
+	case STOPPOD_DEPRECATED:
+		fprintf(stderr, "get abandoned STOPPOD message\n");
+		ret = -1;
+		break;
 	case DESTROYPOD:
+		pod->req_destroy = 1;
 		fprintf(stdout, "get DESTROYPOD message\n");
 		hyper_destroy_pod(pod, 0);
 		return 0;
@@ -1075,10 +1049,10 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 		ret = hyper_exec_cmd((char *)buf->data + 8, len - 8);
 		break;
 	case WRITEFILE:
-		ret = hyper_cmd_write_file((char *)buf->data + 8, len - 8);
+		ret = hyper_cmd_rw_file((char *)buf->data + 8, len - 8, NULL, NULL, WRITEFILE);
 		break;
 	case READFILE:
-		ret = hyper_cmd_read_file((char *)buf->data + 8, len - 8, &datalen, &data);
+		ret = hyper_cmd_rw_file((char *)buf->data + 8, len - 8, &datalen, &data, READFILE);
 		break;
 	case PING:
 	case GETPOD:
@@ -1121,22 +1095,79 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 	return 0;
 }
 
+static int hyper_channel_read(struct hyper_event *he, int efd)
+{
+	struct hyper_buf *buf = &he->rbuf;
+	uint32_t len;
+	uint8_t data[4];
+	int size;
+	int ret;
+
+	fprintf(stdout, "%s\n", __func__);
+
+	if (buf->get < CONTROL_HEADER_SIZE) {
+		size = nonblock_read(he->fd, buf->data + buf->get, CONTROL_HEADER_SIZE - buf->get);
+		if (size < 0) {
+			return size;
+		}
+		if (size > 0) {
+			/* control channel, need ack */
+			hyper_set_be32(data, size);
+			hyper_send_msg(he->fd, NEXT, 4, data);
+		}
+		buf->get += size;
+		if (buf->get < CONTROL_HEADER_SIZE) {
+			return 0;
+		}
+	}
+
+	len = hyper_get_be32(buf->data + CONTROL_HEADER_LENGTH_OFFSET);
+	fprintf(stdout, "get length %" PRIu32"\n", len);
+	// test it with '>=' to leave at least one byte in hyper_channel_handle(),
+	// so that hyper_channel_handle() can convert the data to c-string inplace.
+	if (len >= buf->size) {
+		uint8_t *new_data;
+		fprintf(stderr, "get length %" PRIu32", too long, extend buffer\n", len);
+		new_data = realloc(buf->data, len + 1);
+		if (!new_data) {
+			perror("realloc channel read buffer failed");
+			return -1;
+		}
+		buf->data = new_data;
+		buf->size = len + 1;
+	}
+
+	size = nonblock_read(he->fd, buf->data + buf->get, len - buf->get);
+	if (size < 0) {
+		return size;
+	}
+	if (size > 0) {
+		/* control channel, need ack */
+		hyper_set_be32(data, size);
+		hyper_send_msg(he->fd, NEXT, 4, data);
+	}
+	buf->get += size;
+	if (buf->get < len) {
+		return 0;
+	}
+
+	/* get and consume the whole data */
+	ret = hyper_channel_handle(he, len);
+	buf->get = 0;
+
+	return ret == 0 ? 0 : -1;
+}
+
 static struct hyper_event_ops hyper_channel_ops = {
-	.read		= hyper_event_read,
-	.handle		= hyper_channel_handle,
+	.read		= hyper_channel_read,
 	.rbuf_size	= 10240,
-	.len_offset	= 4,
-	/* TODO: vbox hyper should support channel ack */
-	.ack		= 1,
 };
 
 static struct hyper_event_ops hyper_ttyfd_ops = {
-	.read		= hyper_event_read,
+	.read		= hyper_ttyfd_read,
 	.write		= hyper_event_write,
-	.handle		= hyper_ttyfd_handle,
 	.rbuf_size	= 4096,
 	.wbuf_size	= 10240,
-	.len_offset	= 8,
 };
 
 static int hyper_loop(void)
@@ -1214,14 +1245,14 @@ static int hyper_loop(void)
 
 	while (1) {
 		n = epoll_pwait(ctl.efd, events, MAXEVENTS, -1, &omask);
-		fprintf(stdout, "%s epoll_wait %d\n", __func__, n);
-
 		if (n < 0) {
 			if (errno == EINTR)
 				continue;
 			perror("hyper wait event failed");
 			return -1;
 		}
+		fprintf(stdout, "%s epoll_wait %d\n", __func__, n);
+
 		for (i = 0; i < n; i++) {
 			if (hyper_handle_event(ctl.efd, &events[i]) < 0)
 				return -1;
@@ -1237,14 +1268,6 @@ int main(int argc, char *argv[])
 {
 	char *cmdline, *ctl_serial, *tty_serial;
 
-	if (hyper_mkdir("/dev", 0755) < 0 ||
-	    hyper_mkdir("/sys", 0755) < 0 ||
-	    hyper_mkdir("/sbin", 0755) < 0 ||
-	    hyper_mkdir("/proc", 0755) < 0) {
-		perror("create basic directroy failed");
-		return -1;
-	}
-
 	if (mount("proc", "/proc", "proc", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) == -1) {
 		perror("mount proc failed");
 		return -1;
@@ -1258,12 +1281,12 @@ int main(int argc, char *argv[])
 	}
 
 	if (mount("dev", "/dev", "devtmpfs", MS_NOSUID, NULL) == -1) {
-		perror("mount sysfs failed");
+		perror("mount devtmpfs failed");
 		return -1;
 	}
 
 	if (hyper_mkdir("/dev/pts", 0755) < 0) {
-		perror("create basic directroy failed");
+		perror("create basic directory failed");
 		return -1;
 	}
 
@@ -1272,14 +1295,6 @@ int main(int argc, char *argv[])
 		return -1;
 	}
 
-	symlink("/busybox", "/sh");
-	symlink("/busybox", "/tar");
-	symlink("/busybox", "/sbin/modprobe");
-	symlink("/busybox", "/sbin/depmod");
-	symlink("/iptables", "/sbin/iptables");
-	symlink("/iptables", "/sbin/iptables-restore");
-	symlink("/iptables", "/sbin/iptables-save");
-
 	cmdline = read_cmdline();
 
 	setsid();

src/net.c

@@ -48,7 +48,6 @@ int hyper_send_data(int fd, uint8_t *data, uint32_t len)
 
 	while (length < len) {
 		size = write(fd, data + length, len - length);
-
 		if (size <= 0) {
 			if (errno == EINTR)
 				continue;
@@ -99,7 +98,6 @@ int hyper_get_type(int fd, uint32_t *type)
 
 	while (len < 8) {
 		size = read(fd, buf + len, 8 - len);
-
 		if (size <= 0) {
 			if (errno == EINTR)
 				continue;
@@ -179,7 +177,7 @@ static int addattr_l(struct nlmsghdr *n, int maxlen, int type, void *data, int a
 static int hyper_get_ifindex(char *nic)
 {
 	int fd, ifindex = -1;
-	char path[512], buf[4];
+	char path[512], buf[8];
 
 	fprintf(stdout, "net device %s\n", nic);
 	sprintf(path, "/sys/class/net/%s/ifindex", nic);
@@ -192,7 +190,7 @@ static int hyper_get_ifindex(char *nic)
 	}
 
 	memset(buf, 0, sizeof(buf));
-	if (read(fd, buf, sizeof(buf)) <= 0) {
+	if (read(fd, buf, sizeof(buf) - 1) <= 0) {
 		perror("can read open file");
 		goto out;
 	}
@@ -430,8 +428,13 @@ static int hyper_setup_route(struct rtnl_handle *rth,
 	}
 
 	if (rt->device) {
-		rt->ifindex = hyper_get_ifindex(rt->device);
-		if (addattr_l(&req.n, sizeof(req), RTA_OIF, &rt->ifindex, 4)) {
+		int ifindex = hyper_get_ifindex(rt->device);
+		if (ifindex < 0) {
+			fprintf(stderr, "failed to get the ifindix of %s\n", rt->device);
+			return -1;
+		}
+
+		if (addattr_l(&req.n, sizeof(req), RTA_OIF, &ifindex, 4)) {
 			fprintf(stderr, "setup oif attr failed\n");
 			return -1;
 		}
@@ -513,7 +516,13 @@ static int hyper_cleanup_route(struct rtnl_handle *rth, struct hyper_route *rt)
 	}
 
 	if (rt->device) {
-		if (addattr_l(&req.n, sizeof(req), RTA_OIF, &rt->ifindex, 4)) {
+		int ifindex = hyper_get_ifindex(rt->device);
+		if (ifindex < 0) {
+			fprintf(stderr, "failed to get the ifindix of %s\n", rt->device);
+			return -1;
+		}
+
+		if (addattr_l(&req.n, sizeof(req), RTA_OIF, &ifindex, 4)) {
 			fprintf(stderr, "setup oif attr failed\n");
 			return -1;
 		}
@@ -556,6 +565,44 @@ static int hyper_cleanup_route(struct rtnl_handle *rth, struct hyper_route *rt)
 	return 0;
 }
 
+static int hyper_set_interface_name(struct rtnl_handle *rth,
+				int ifindex,
+				char *new_device_name)
+{
+	struct {
+                struct nlmsghdr n;
+                struct ifinfomsg i;
+                char buf[1024];
+        } req;
+
+	if (ifindex < 0 || !new_device_name) {
+		return -1;
+	}
+
+	memset(&req, 0, sizeof(req));
+	req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
+	req.n.nlmsg_flags = NLM_F_REQUEST;
+	req.n.nlmsg_type = RTM_SETLINK;
+
+	req.i.ifi_family = AF_UNSPEC;
+	req.i.ifi_change = 0xFFFFFFFF;
+	req.i.ifi_index = ifindex;
+
+	if (addattr_l(&req.n, sizeof(req), IFLA_IFNAME, 
+			new_device_name,
+			strlen(new_device_name) + 1)) {
+                fprintf(stderr, "setup attr failed\n");
+                return -1;
+        }
+
+	if (rtnl_talk(rth, &req.n, 0, 0, NULL) < 0) {
+		perror("rtnl_talk failed");
+		return -1;
+	}
+
+	return 0;
+}
+
 static int hyper_setup_interface(struct rtnl_handle *rth,
 			       struct hyper_interface *iface)
 {
@@ -566,6 +613,7 @@ static int hyper_setup_interface(struct rtnl_handle *rth,
 		struct ifaddrmsg ifa;
 		char buf[256];
 	} req;
+	int ifindex;
 
 	if (!(iface->device && iface->ipaddr && iface->mask)) {
 		fprintf(stderr, "interface information incorrect\n");
@@ -578,8 +626,13 @@ static int hyper_setup_interface(struct rtnl_handle *rth,
 	req.n.nlmsg_type = RTM_NEWADDR;
 	req.ifa.ifa_family = AF_INET;
 
-	iface->ifindex = hyper_get_ifindex(iface->device);
-	req.ifa.ifa_index = iface->ifindex;
+	ifindex = hyper_get_ifindex(iface->device);
+	if (ifindex < 0) {
+		fprintf(stderr, "failed to get the ifindix of %s\n", iface->device);
+		return -1;
+	}
+
+	req.ifa.ifa_index = ifindex;
 	req.ifa.ifa_scope = 0;
 
 	if (get_addr_ipv4((uint8_t *)&data, iface->ipaddr) <= 0) {
@@ -604,8 +657,13 @@ static int hyper_setup_interface(struct rtnl_handle *rth,
 		return -1;
 	}
 
-	if (hyper_up_nic(rth, iface->ifindex) < 0) {
-		fprintf(stderr, "up device %d failed\n", iface->ifindex);
+	if (iface->new_device_name && strcmp(iface->new_device_name, iface->device)) {
+		fprintf(stdout, "Setting interface name to %s\n", iface->new_device_name);
+		hyper_set_interface_name(rth, ifindex, iface->new_device_name);
+	}
+
+	if (hyper_up_nic(rth, ifindex) < 0) {
+		fprintf(stderr, "up device %d failed\n", ifindex);
 		return -1;
 	}
 
@@ -622,6 +680,7 @@ static int hyper_cleanup_interface(struct rtnl_handle *rth,
 		struct ifaddrmsg ifa;
 		char buf[256];
 	} req;
+	int ifindex;
 
 	if (!(iface->device && iface->ipaddr && iface->mask)) {
 		fprintf(stderr, "interface information incorrect\n");
@@ -634,7 +693,13 @@ static int hyper_cleanup_interface(struct rtnl_handle *rth,
 	req.n.nlmsg_type = RTM_DELADDR;
 	req.ifa.ifa_family = AF_INET;
 
-	req.ifa.ifa_index = iface->ifindex;
+	ifindex = hyper_get_ifindex(iface->device);
+	if (ifindex < 0) {
+		fprintf(stderr, "failed to get the ifindix of %s\n", iface->device);
+		return -1;
+	}
+
+	req.ifa.ifa_index = ifindex;
 	req.ifa.ifa_scope = 0;
 
 	if (get_addr_ipv4((uint8_t *)&data, iface->ipaddr) <= 0) {
@@ -664,8 +729,8 @@ static int hyper_cleanup_interface(struct rtnl_handle *rth,
 		return 0;
 	}
 
-	if (hyper_down_nic(rth, iface->ifindex) < 0) {
-		fprintf(stderr, "up device %d failed\n", iface->ifindex);
+	if (hyper_down_nic(rth, ifindex) < 0) {
+		fprintf(stderr, "up device %d failed\n", ifindex);
 		return -1;
 	}
 
@@ -891,6 +956,9 @@ void hyper_cleanup_dns(struct hyper_pod *pod)
 {
 	int fd, i;
 
+	if (pod->dns == NULL)
+		return;
+
 	for (i = 0; i < pod->d_num; i++) {
 		free(pod->dns[i]);
 	}

src/net.h

@@ -18,26 +18,17 @@ struct rtnl_handle {
 	__u32 dump;
 };
 
-typedef struct {
-	__u8 family;
-	__u8 bytelen;
-	__s16 bitlen;
-	__u32 flags;
-	__u32 data[8];
-} inet_prefix;
-
 struct hyper_interface {
 	char		*device;
-	int		ifindex;
 	char		*ipaddr;
 	char		*mask;
+	char            *new_device_name;
 };
 
 struct hyper_route {
 	char		*dst;
 	char		*gw;
 	char		*device;
-	int		ifindex;
 };
 
 struct hyper_pod;

src/parse.c

@@ -184,8 +184,8 @@ static void container_cleanup_exec(struct hyper_exec *exec)
 {
 	int i;
 
-	free(exec->id);
-	exec->id = NULL;
+	free(exec->container_id);
+	exec->container_id = NULL;
 
 	free(exec->user);
 	exec->user = NULL;
@@ -639,9 +639,6 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container *
 	c->exec.stdoutev.fd = -1;
 	c->exec.stderrev.fd = -1;
 	c->exec.ptyfd = -1;
-	c->exec.stdinfd = -1;
-	c->exec.stdoutfd = -1;
-	c->exec.stderrfd = -1;
 	c->ns = -1;
 	INIT_LIST_HEAD(&c->list);
 
@@ -653,7 +650,7 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container *
 		fprintf(stdout, "%d name %s\n", i, json_token_str(json, t));
 		if (json_token_streq(json, t, "id") && t->size == 1) {
 			c->id = (json_token_str(json, &toks[++i]));
-			c->exec.id = strdup(c->id);
+			c->exec.container_id = strdup(c->id);
 			fprintf(stdout, "container id %s\n", c->id);
 			i++;
 		} else if (json_token_streq(json, t, "rootfs") && t->size == 1) {
@@ -776,6 +773,9 @@ static int hyper_parse_interface(struct hyper_interface *iface,
 		} else if (json_token_streq(json, &toks[i], "netMask")) {
 			iface->mask = (json_token_str(json, &toks[++i]));
 			fprintf(stdout, "net mask is %s\n", iface->mask);
+		} else if (json_token_streq(json, &toks[i], "newDeviceName")) {
+			iface->new_device_name = (json_token_str(json, &toks[++i]));
+			fprintf(stdout, "new interface name is %s\n", iface->new_device_name);
 		} else {
 			fprintf(stderr, "get unknown section %s in interfaces\n",
 				json_token_str(json, &toks[i]));
@@ -789,6 +789,7 @@ fail:
 	free(iface->device);
 	free(iface->ipaddr);
 	free(iface->mask);
+	free(iface->new_device_name);
 	return -1;
 }
 
@@ -1146,8 +1147,6 @@ realloc:
 		goto out;
 	}
 
-	pod->policy = POLICY_NEVER;
-
 	fprintf(stdout, "jsmn parse successed, n is %d\n", n);
 	next = 0;
 	for (i = 0; i < n;) {
@@ -1193,13 +1192,8 @@ realloc:
 			fprintf(stdout, "hostname is %s\n", pod->hostname);
 			i++;
 		} else if (json_token_streq(json, t, "restartPolicy") && t->size == 1) {
-			i++;
-			if (json_token_streq(json, &toks[i], "always") && toks[i].size == 1)
-				pod->policy = POLICY_ALWAYS;
-			else if (json_token_streq(json, &toks[i], "onFailure") && toks[i].size == 1)
-				pod->policy = POLICY_ONFAILURE;
-			fprintf(stdout, "restartPolicy is %" PRIu8 "\n", pod->policy);
-			i++;
+			i += 2;
+			fprintf(stdout, "restartPolicy is deprecated\n");
 		} else if (json_token_streq(json, t, "portmappingWhiteLists") && t->size == 1) {
 			next = hyper_parse_portmapping_whitelist(pod, json, &toks[++i]);
 			if (next < 0)
@@ -1257,68 +1251,6 @@ fail:
 	return NULL;
 }
 
-int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length)
-{
-	int i, n, ret = -1;
-	jsmn_parser p;
-	int toks_num = 10;
-	jsmntok_t *toks = NULL;
-
-	memset(ws, 0, sizeof(*ws));
-realloc:
-	toks = realloc(toks, toks_num * sizeof(jsmntok_t));
-	if (toks == NULL) {
-		fprintf(stderr, "allocate tokens for winsize failed\n");
-		goto out;
-	}
-
-	jsmn_init(&p);
-
-	n = jsmn_parse(&p, json, length,  toks, toks_num);
-	if (n < 0) {
-		fprintf(stdout, "jsmn parse failed, n is %d\n", n);
-		if (n == JSMN_ERROR_NOMEM) {
-			toks_num *= 2;
-			goto realloc;
-		}
-
-		goto out;
-	}
-
-	for (i = 0; i < n; i++) {
-		jsmntok_t *t = &toks[i];
-
-		if (t->type != JSMN_STRING)
-			continue;
-
-		if (i++ == n)
-			goto out;
-
-		if (json_token_streq(json, t, "seq")) {
-			if (toks[i].type != JSMN_PRIMITIVE)
-				goto out;
-			ws->seq = json_token_ll(json, &toks[i]);
-		} else if (json_token_streq(json, t, "row")) {
-			if (toks[i].type != JSMN_PRIMITIVE)
-				goto out;
-			ws->row = json_token_int(json, &toks[i]);
-		} else if (json_token_streq(json, t, "column")) {
-			if (toks[i].type != JSMN_PRIMITIVE)
-				goto out;
-			ws->column = json_token_int(json, &toks[i]);
-		} else {
-			fprintf(stderr, "get unknown section %s in winsize\n",
-				json_token_str(json, t));
-			goto out;
-		}
-	}
-
-	ret = 0;
-out:
-	free(toks);
-	return ret;
-}
-
 struct hyper_exec *hyper_parse_execcmd(char *json, int length)
 {
 	int i, j, n;
@@ -1353,9 +1285,6 @@ realloc:
 	}
 
 	exec->ptyfd = -1;
-	exec->stdinfd = -1;
-	exec->stdoutfd = -1;
-	exec->stderrfd = -1;
 	exec->stdinev.fd = -1;
 	exec->stdoutev.fd = -1;
 	exec->stderrev.fd = -1;
@@ -1365,8 +1294,8 @@ realloc:
 		jsmntok_t *t = &toks[i];
 
 		if (json_token_streq(json, t, "container")) {
-			exec->id = (json_token_str(json, &toks[++i]));
-			fprintf(stdout, "get container %s\n", exec->id);
+			exec->container_id = (json_token_str(json, &toks[++i]));
+			fprintf(stdout, "get container %s\n", exec->container_id);
 		} else if (json_token_streq(json, t, "process") && t->size == 1) {
 			j = hyper_parse_process(exec, json, &toks[++i]);
 			if (j < 0)
@@ -1375,7 +1304,7 @@ realloc:
 		}
 	}
 
-	if (exec->id == NULL || strlen(exec->id) == 0) {
+	if (exec->container_id == NULL || strlen(exec->container_id) == 0) {
 		fprintf(stderr, "execcmd format error, has no container id\n");
 		goto fail;
 	}
@@ -1395,7 +1324,7 @@ fail:
 	goto out;
 }
 
-int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length)
+int hyper_parse_file_command(struct file_command *cmd, char *json, int length)
 {
 	int i, n, ret = -1;
 
@@ -1403,84 +1332,11 @@ int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length
 	int toks_num = 10;
 	jsmntok_t *toks = NULL;
 
-	memset(writter, 0, sizeof(*writter));
+	memset(cmd, 0, sizeof(*cmd));
 
 	toks = calloc(toks_num, sizeof(jsmntok_t));
 	if (toks == NULL) {
-		fprintf(stderr, "fail to allocate tokens for write file cmd\n");
-		goto fail;
-	}
-
-	jsmn_init(&p);
-	n = jsmn_parse(&p, json, length,  toks, toks_num);
-	/* Must be json first */
-	if (n <= 0) {
-		fprintf(stdout, "jsmn parse failed, n is %d\n", n);
-		goto fail;
-	}
-
-	writter->len = length - toks[0].end;
-	writter->data = malloc(writter->len);
-
-	if (writter->data == NULL) {
-		fprintf(stderr, "fail to allocate memory for writter data\n");
-		goto fail;
-	}
-
-	memcpy(writter->data, json + toks[0].end, writter->len);
-	fprintf(stdout, "writefile get data len %d %s\n", writter->len, writter->data);
-
-	for (i = 0; i < n; i++) {
-		jsmntok_t *t = &toks[i];
-
-		if (t->type != JSMN_STRING)
-			continue;
-
-		if (i++ == n)
-			goto fail;
-
-		if (json_token_streq(json, t, "container")) {
-			writter->id = (json_token_str(json, &toks[i]));
-			fprintf(stdout, "writefile get container %s\n", writter->id);
-		} else if (json_token_streq(json, t, "file")) {
-			writter->file = (json_token_str(json, &toks[i]));
-			fprintf(stdout, "writefile get file %s\n", writter->file);
-		} else {
-			fprintf(stderr, "get unknown section %s in writefile\n",
-				json_token_str(json, t));
-			goto fail;
-		}
-	}
-
-	if (writter->id == NULL || writter->file == NULL) {
-		fprintf(stderr, "writefile format incorrect\n");
-		goto fail;
-	}
-
-	ret = 0;
-out:
-	free(toks);
-	return ret;
-fail:
-	free(writter->id);
-	free(writter->file);
-	free(writter->data);
-	goto out;
-}
-
-int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length)
-{
-	int i, n, ret = -1;
-
-	jsmn_parser p;
-	int toks_num = 10;
-	jsmntok_t *toks = NULL;
-
-	memset(reader, 0, sizeof(*reader));
-
-	toks = calloc(toks_num, sizeof(jsmntok_t));
-	if (toks == NULL) {
-		fprintf(stderr, "fail to allocate tokens for read file cmd\n");
+		fprintf(stderr, "fail to allocate tokens for file cmd\n");
 		ret = -1;
 		goto fail;
 	}
@@ -1503,20 +1359,20 @@ int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length)
 			goto fail;
 
 		if (json_token_streq(json, t, "container")) {
-			reader->id = (json_token_str(json, &toks[i]));
-			fprintf(stdout, "readfile get container %s\n", reader->id);
+			cmd->id = (json_token_str(json, &toks[i]));
+			fprintf(stdout, "file cmd get container %s\n", cmd->id);
 		} else if (json_token_streq(json, t, "file")) {
-			reader->file = (json_token_str(json, &toks[i]));
-			fprintf(stdout, "readfile get file %s\n", reader->file);
+			cmd->file = (json_token_str(json, &toks[i]));
+			fprintf(stdout, "file cmd get file %s\n", cmd->file);
 		} else {
-			fprintf(stdout, "get unknown section %s in readfile\n",
+			fprintf(stdout, "get unknown section %s in file cmd\n",
 				json_token_str(json, t));
 			goto fail;
 		}
 	}
 
-	if (reader->id == NULL || reader->file == NULL) {
-		fprintf(stderr, "readfile format incorrect\n");
+	if (cmd->id == NULL || cmd->file == NULL) {
+		fprintf(stderr, "file cmd format incorrect\n");
 		goto fail;
 	}
 
@@ -1525,10 +1381,10 @@ out:
 	free(toks);
 	return ret;
 fail:
-	free(reader->id);
-	reader->id = NULL;
-	free(reader->file);
-	reader->file = NULL;
+	free(cmd->id);
+	cmd->id = NULL;
+	free(cmd->file);
+	cmd->file = NULL;
 	goto out;
 }
 

src/parse.h

@@ -10,8 +10,7 @@ struct hyper_exec *hyper_parse_execcmd(char *json, int length);
 char *json_token_str(char *js, jsmntok_t *t);
 int json_token_streq(char *js, jsmntok_t *t, char *s);
 int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length);
-int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length);
-int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length);
+int hyper_parse_file_command(struct file_command *cmd, char *json, int length);
 struct hyper_container *hyper_parse_new_container(struct hyper_pod *pod, char *json, int length);
 void hyper_free_container(struct hyper_container *c);
 struct hyper_interface *hyper_parse_setup_interface(char *json, int length);

src/portmapping.c

@@ -16,7 +16,7 @@
 
 int hyper_init_modules() 
 {
-	int status = hyper_cmd("/sbin/depmod");
+	int status = hyper_cmd("depmod");
 	if (status != 0) {
 		fprintf(stderr, "depmod failed, status: %d\n", status);
 		return -1;
@@ -32,10 +32,10 @@ int hyper_setup_iptables_rule(struct ipt_rule rule)
 	int check = -1;
 
 	if (rule.rule != NULL) {
-		sprintf(check_cmd, "/sbin/iptables -t %s -C %s %s", rule.table, rule.chain, rule.rule);
-		sprintf(cmd, "/sbin/iptables -t %s %s %s %s", rule.table, rule.op, rule.chain, rule.rule);
+		sprintf(check_cmd, "iptables -t %s -C %s %s", rule.table, rule.chain, rule.rule);
+		sprintf(cmd, "iptables -t %s %s %s %s", rule.table, rule.op, rule.chain, rule.rule);
 	} else {
-		sprintf(cmd, "/sbin/iptables -t %s %s %s", rule.table, rule.op, rule.chain);
+		sprintf(cmd, "iptables -t %s %s %s", rule.table, rule.op, rule.chain);
 	}
 
 	if (strlen(check_cmd) > 0) {
@@ -66,6 +66,9 @@ int hyper_setup_iptables_rule(struct ipt_rule rule)
 // initialize modules and iptables chains
 int hyper_setup_portmapping(struct hyper_pod *pod)
 {
+	const char *connmax = "10485760";
+	const char *timeout = "300";
+
 	if (pod->portmap_white_lists == NULL || (pod->portmap_white_lists->i_num == 0 &&
 			pod->portmap_white_lists->e_num == 0)) {
 		return 0;
@@ -148,6 +151,16 @@ int hyper_setup_portmapping(struct hyper_pod *pod)
 		}
 	}
 
+	/* portmapping enables nf_conntrack by default, should blow up nf_conntack_max to make sure
+	 * nf_conntrack is available for connections. */
+	if (hyper_write_file("/proc/sys/net/nf_conntrack_max", connmax, strlen(connmax)) < 0) {
+		fprintf(stderr, "sysctl: setup default nf_conntrack_max(%s) failed\n", connmax);
+	}
+
+	if (hyper_write_file("/proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established", timeout, strlen(timeout)) < 0) {
+		fprintf(stderr, "sysctl: setup default nf_conntrack_tcp_timeout_established(%s) failed\n", timeout);
+	}
+
 	return 0;
 }
 
@@ -258,12 +271,28 @@ int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_po
 		return 0;
 	}
 
+	// only allow network request from internal white list
+	int i = 0, j = 0;
+	char rule[128] = {0};
+	for (j=0; j<pod->portmap_white_lists->i_num; j++) {
+		sprintf(rule, "-s %s -j ACCEPT",
+			pod->portmap_white_lists->internal_networks[j]);
+		struct ipt_rule accept_rule = {
+			.table = "filter",
+			.op = "-I",
+			.chain = "hyperstart-INPUT",
+			.rule = rule,
+		};
+		if (hyper_setup_iptables_rule(accept_rule)<0) {
+			fprintf(stderr, "setup accept_rule '%s' failed\n", rule);
+			return -1;
+		}
+	}
+
 	if (c->ports_num == 0) {
 		return 0;
 	}
 
-	int i = 0, j = 0;
-	char rule[128] = {0};
 	char *network = NULL;
 	for (i=0; i<c->ports_num; i++) {
 		// setup port mapping only if host_port is set
@@ -309,25 +338,6 @@ int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_po
 				}
 			}
 		}
-
-		// only allow network request from white list
-		for (j=0; j<pod->portmap_white_lists->i_num; j++) {
-			sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
-				pod->portmap_white_lists->internal_networks[j],
-				c->ports[i].protocol,
-				c->ports[i].protocol,
-				c->ports[i].container_port);
-			struct ipt_rule accept_rule = {
-				.table = "filter",
-				.op = "-I",
-				.chain = "hyperstart-INPUT",
-				.rule = rule,
-			};
-			if (hyper_setup_iptables_rule(accept_rule)<0) {
-				fprintf(stderr, "setup accept_rule '%s' failed\n", rule);
-				return -1;
-			}
-		}
 	}
 
 	return 0;
@@ -340,12 +350,26 @@ void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper
 		return;
 	}
 
+	int i = 0, j = 0;
+	char rule[128] = {0};
+	for (j=0; j<pod->portmap_white_lists->i_num; j++) {
+		sprintf(rule, "-s %s -j ACCEPT",
+			pod->portmap_white_lists->internal_networks[j]);
+		struct ipt_rule accept_rule = {
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = rule,
+		};
+		if (hyper_setup_iptables_rule(accept_rule)<0) {
+			fprintf(stderr, "cleanup accept_rule '%s' failed\n", rule);
+		}
+	}
+
 	if (c->ports_num == 0) {
 		return;
 	}
 
-	int i = 0, j = 0;
-	char rule[128] = {0};
 	char *network = NULL;
 	for (i=0; i<c->ports_num; i++) {
 		// clean up port mapping only if host_port is set
@@ -389,22 +413,5 @@ void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper
 				}
 			}
 		}
-
-		for (j=0; j<pod->portmap_white_lists->i_num; j++) {
-			sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
-				pod->portmap_white_lists->internal_networks[j],
-				c->ports[i].protocol,
-				c->ports[i].protocol,
-				c->ports[i].container_port);
-			struct ipt_rule accept_rule = {
-				.table = "filter",
-				.op = "-D",
-				.chain = "hyperstart-INPUT",
-				.rule = rule,
-			};
-			if (hyper_setup_iptables_rule(accept_rule)<0) {
-				fprintf(stderr, "cleanup accept_rule '%s' failed\n", rule);
-			}
-		}
 	}
 }

src/util.c

@@ -71,7 +71,7 @@ int hyper_list_dir(char *path)
 int hyper_copy_dir(char *src, char *dest)
 {
 	char cmd[512];
-	snprintf(cmd, sizeof(cmd), "/tar cf - -C %s . | /tar fx - -C %s", src, dest);
+	snprintf(cmd, sizeof(cmd), "tar cf - -C %s . | tar fx - -C %s", src, dest);
 
 	return hyper_cmd(cmd);
 }
@@ -82,7 +82,7 @@ void hyper_sync_time_hctosys() {
 	if (pid < 0) {
 		perror("fail to fork to copy directory");
 	} else if (pid == 0) {
-		execlp("/busybox", "hwclock", "-s", NULL);
+		execlp("hwclock", "hwclock", "-s", NULL);
 		perror("exec hwclock -s command failed");
 		exit(-1);
 	}
@@ -415,13 +415,12 @@ int hyper_open_channel(char *channel, int mode)
 {
 	struct termios term;
 	int fd = open(channel, O_RDWR | O_CLOEXEC | mode);
-	fprintf(stdout, "open %s get %d\n", channel, fd);
-
 	if (fd < 0) {
 		perror("fail to open channel device");
 		return -1;
 	}
 
+	fprintf(stdout, "open %s get %d\n", channel, fd);
 	bzero(&term, sizeof(term));
 
 	cfmakeraw(&term);
@@ -512,7 +511,7 @@ int hyper_open_channel(char *channel, int mode)
 
 	num = scandir("/sys/class/virtio-ports/", &list, NULL, NULL);
 	if (num < 0) {
-		perror("scan /sys/calss/virtio-ports/ failed");
+		perror("scan /sys/class/virtio-ports/ failed");
 		return -1;
 	}
 
@@ -538,7 +537,6 @@ int hyper_open_channel(char *channel, int mode)
 		fd = -1;
 
 		if (strncmp(name, channel, strlen(channel))) {
-			fprintf(stderr, "channel %s, directory %s\n", channel, name);
 			continue;
 		}
 
@@ -565,16 +563,6 @@ int hyper_insmod(char *module)
 }
 #endif
 
-int hyper_open_serial_dev(char *tty)
-{
-	int fd = open(tty, O_RDWR | O_CLOEXEC | O_NOCTTY);
-
-	if (fd < 0)
-		perror("fail to open tty device");
-
-	return fd;
-}
-
 int hyper_setfd_cloexec(int fd)
 {
 	int flags = fcntl(fd, F_GETFD);
@@ -673,9 +661,8 @@ static void hyper_unmount_all(void)
 	sync();
 }
 
-void hyper_shutdown(int error)
+void hyper_shutdown()
 {
-	hyper_send_msg_block(ctl.chan.fd, error?ERROR:ACK, 0, NULL);
 	hyper_unmount_all();
 	reboot(LINUX_REBOOT_CMD_POWER_OFF);
 }
@@ -704,8 +691,30 @@ int hyper_cmd(char *cmd)
 		return -1;
 	} else {
 		fprintf(stdout, "executing cmd %s\n", cmd);
-		execlp("/busybox", "sh", "-c", cmd, NULL);
+		execlp("sh", "sh", "-c", cmd, NULL);
 	}
 
 	return -1;
 }
+
+ssize_t nonblock_read(int fd, void *buf, size_t count)
+{
+	ssize_t len = 0, ret = 0;
+
+	while (len < count) {
+		ret = read(fd, buf + len, count - len);
+		if (ret <= 0) {
+			if (errno == EINTR) {
+				continue;
+			}
+			if (errno == EAGAIN) {
+				ret = 0;
+			}
+			break;
+		}
+
+		len += ret;
+	}
+
+	return len > 0 ? len : ret;
+}

src/util.h

@@ -30,14 +30,14 @@ void hyper_filize(char *hyper_path);
 int hyper_mkdir(char *path, mode_t mode);
 int hyper_write_file(const char *path, const char *value, size_t len);
 int hyper_open_channel(char *channel, int mode);
-int hyper_open_serial_dev(char *tty);
 int hyper_setfd_cloexec(int fd);
 int hyper_setfd_block(int fd);
 int hyper_setfd_nonblock(int fd);
 int hyper_socketpair(int domain, int type, int protocol, int sv[2]);
-void hyper_shutdown(int ack);
+void hyper_shutdown();
 int hyper_insmod(char *module);
 struct passwd *hyper_getpwnam(const char *name);
 struct group *hyper_getgrnam(const char *name);
 int hyper_getgrouplist(const char *user, gid_t group, gid_t *groups, int *ngroups);
+ssize_t nonblock_read(int fd, void *buf, size_t count);
 #endif