Merge pull request #150 from laijs/v0.6.2

bump version to 0.6.2

.gitignore

@@ -14,6 +14,7 @@ build/cbfs.rom
 build/root/*
 build/*iso
 build/daemon
+build/kernel_config.old
 build/.*
 src/.*
 Makefile

Makefile.am

@@ -4,3 +4,5 @@ initrd-local:
 	@echo finish make initrd
 cbfs-local:
 	@echo finish make cbfs
+kernel-local:
+	@echo finish make kernel

README.md

@@ -1,6 +1,6 @@
-# The init Task for Hyper
+# The init Task for HyperContainer
 
-You can get the binary installer of Hyper and HyperStart through [The Hyper Page](https://github.com/hyperhq/hyper)
+You can get the binary installer of HyperContainer and HyperStart through [The Hyper Page](https://github.com/hyperhq/hyperd)
 
 ## Build from source 
 

build/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos:7
+MAINTAINER Hyper Developers <dev@hyper.sh>
+
+RUN yum install -y patch gcc ncurses-devel make openssl-devel bc
+
+ENV KERNEL_VERSION 4.4.12
+ENV LOCALVERSION -hyper
+ENV KERNEL_RELEASE ${KERNEL_VERSION}${LOCALVERSION}
+
+ENV KBUILD_BUILD_USER    dev
+ENV KBUILD_BUILD_HOST    hyper.sh
+ENV KBUILD_BUILD_VERSION 1
+
+RUN mkdir /root/build/ && mkdir /root/build/result/
+RUN curl -fSL https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-${KERNEL_VERSION}.tar.gz | tar -zx -C /root/build
+
+COPY kernel_config /root/build/linux-${KERNEL_VERSION}/.config
+COPY kernel_patch/ /root/build/kernel_patch/
+
+RUN cd /root/build/linux-${KERNEL_VERSION}/ && for patch in /root/build/kernel_patch/*.patch; do patch -p1 <$patch || exit 1; done
+RUN cd /root/build/linux-${KERNEL_VERSION}/ && make silentoldconfig && make -j 8
+
+# install to /root/build/result/ so that we can get them from it
+RUN cp /root/build/linux-${KERNEL_VERSION}/arch/x86_64/boot/bzImage /root/build/result/kernel
+RUN mkdir /root/build/result/modules &&\
+    cd /root/build/linux-${KERNEL_VERSION}/ && make modules_install INSTALL_MOD_PATH="/root/build/result/modules" &&\
+    cd /root/build/result/modules/lib/modules/ && rm -f ${KERNEL_RELEASE}/{build,source} &&\
+    tar -cf /root/build/result/modules.tar ${KERNEL_RELEASE}/ && rm -rf /root/build/result/modules
+RUN cp /root/build/linux-${KERNEL_VERSION}/.config /root/build/result/kernel_config

build/Makefile.am

@@ -10,3 +10,10 @@ endif
 
 cbfs-local:
 	bash ./make-initrd.sh cbfs
+
+kernel-local:
+	hyperctl build -t hyperstart-dev:latest .
+	hyperctl run --rm hyperstart-dev:latest cat /root/build/result/kernel >kernel.new && mv -f kernel.new kernel
+	hyperctl run --rm hyperstart-dev:latest cat /root/build/result/modules.tar >modules.tar
+	mv kernel_config kernel_config.old
+	hyperctl run --rm hyperstart-dev:latest cat /root/build/result/kernel_config >kernel_config

build/kernel_patch/0001-HACK-9P-always-use-cached-inode-to-fill-in-v9fs_vfs_.patch

@@ -0,0 +1,43 @@
+From 084b4a5aa4b90975f7ac6b4714a6b449430ef4c4 Mon Sep 17 00:00:00 2001
+From: Peng Tao <bergwolf@gmail.com>
+Date: Wed, 4 May 2016 12:01:22 +0800
+Subject: [PATCH] HACK: 9P: always use cached inode to fill in v9fs_vfs_getattr
+
+So that if in cache=none mode, we don't have to lookup server that
+might not support open-unlink-fstat operation.
+
+Signed-off-by: Peng Tao <bergwolf@gmail.com>
+---
+ fs/9p/vfs_inode.c      | 2 +-
+ fs/9p/vfs_inode_dotl.c | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c
+index 5110785..43ebb31 100644
+--- a/fs/9p/vfs_inode.c
++++ b/fs/9p/vfs_inode.c
+@@ -1059,7 +1059,7 @@ v9fs_vfs_getattr(struct vfsmount *mnt, struct dentry *dentry,
+ 
+ 	p9_debug(P9_DEBUG_VFS, "dentry: %p\n", dentry);
+ 	v9ses = v9fs_dentry2v9ses(dentry);
+-	if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) {
++	if (!d_really_is_negative(dentry) || v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) {
+ 		generic_fillattr(d_inode(dentry), stat);
+ 		return 0;
+ 	}
+diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
+index cb899af..b1e4205 100644
+--- a/fs/9p/vfs_inode_dotl.c
++++ b/fs/9p/vfs_inode_dotl.c
+@@ -479,7 +479,7 @@ v9fs_vfs_getattr_dotl(struct vfsmount *mnt, struct dentry *dentry,
+ 
+ 	p9_debug(P9_DEBUG_VFS, "dentry: %p\n", dentry);
+ 	v9ses = v9fs_dentry2v9ses(dentry);
+-	if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) {
++	if (!d_really_is_negative(dentry) || v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) {
+ 		generic_fillattr(d_inode(dentry), stat);
+ 		return 0;
+ 	}
+-- 
+2.5.0
+

build/make-initrd.sh

@@ -1,18 +1,22 @@
 #!/bin/bash
 
-rm -rf root
-mkdir root
+rm -rf /tmp/hyperstart-rootfs
+mkdir -p /tmp/hyperstart-rootfs/lib /tmp/hyperstart-rootfs/lib64 /tmp/hyperstart-rootfs/lib/modules
 
-cp ../src/init ./root
+cp ../src/init /tmp/hyperstart-rootfs
+cp busybox /tmp/hyperstart-rootfs
+cp iptables /tmp/hyperstart-rootfs
+cp libm.so.6 /tmp/hyperstart-rootfs/lib64/
+tar -xf modules.tar -C /tmp/hyperstart-rootfs/lib/modules
 
-ldd ./root/init | while read line
+ldd /tmp/hyperstart-rootfs/init | while read line
 do
 	arr=(${line// / })
 
 	for lib in ${arr[@]}
 	do
 		if [ "${lib:0:1}" = "/" ]; then
-			dir=root`dirname $lib`
+			dir=/tmp/hyperstart-rootfs`dirname $lib`
 			mkdir -p "${dir}"
 			cp -f $lib $dir
 		fi
@@ -21,13 +25,13 @@ done
 
 if [ "$1"x = "vbox"x ]; then
 	echo "build initrd for vbox"
-	cp ./vbox/driver/* ./root
+	cp ./vbox/driver/* /tmp/hyperstart-rootfs
 fi
 
-cd ./root && find . | cpio -H newc -o | gzip -9 > ../hyper-initrd.img
+( cd /tmp/hyperstart-rootfs && find . | cpio -H newc -o | gzip -9 ) > ./hyper-initrd.img
 
 cd ../
-rm -rf ./root
+rm -rf /tmp/hyperstart-rootfs
 
 if [ "$1"x = "cbfs"x ]; then
 	echo "build cbfs"
@@ -36,7 +40,7 @@ if [ "$1"x = "cbfs"x ]; then
 
 	mkdir .cbfs
 	dd if=/dev/zero of=.cbfs/boot.bin bs=4096 count=1
-	cbfstool .cbfs/cbfs.rom create -s 4096k -B .cbfs/boot.bin -m x86  0x1000
+	cbfstool .cbfs/cbfs.rom create -s 8128k -B .cbfs/boot.bin -m x86  0x1000
 	cbfstool .cbfs/cbfs.rom add -f kernel -n vmlinuz -t raw
 	cbfstool .cbfs/cbfs.rom add -f hyper-initrd.img -n initrd -t raw
 	echo 'console=ttyS0 panic=1 no_timer_check' > .cbfs/cmdline

configure.ac

@@ -2,9 +2,9 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.69])
-AC_INIT([hyperstart], [0.3], [www.hyper.sh])
+AC_INIT([hyperstart], [0.6.2], [www.hyper.sh])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects])
-AM_EXTRA_RECURSIVE_TARGETS([initrd cbfs])
+AM_EXTRA_RECURSIVE_TARGETS([initrd cbfs kernel])
 AC_CONFIG_SRCDIR([src/init.c])
 AC_CONFIG_HEADERS([config.h])
 

src/Makefile.am

@@ -1,3 +1,3 @@
 AM_CFLAGS = -Wall
 bin_PROGRAMS=init
-init_SOURCES=init.c jsmn.c net.c util.c parse.c container.c exec.c event.c
+init_SOURCES=init.c jsmn.c net.c util.c parse.c parson.c container.c exec.c event.c portmapping.c

src/container.c

@@ -7,6 +7,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <sys/utsname.h>
 #include <unistd.h>
 #include <mntent.h>
 #include <signal.h>
@@ -19,6 +20,79 @@
 #include "parse.h"
 #include "syscall.h"
 
+static int container_populate_volume(char *src, char *dest)
+{
+	struct stat st;
+
+	fprintf(stdout, "populate volumes from %s to %s\n", src, dest);
+	/* FIXME: check if has data in volume, (except lost+found) */
+
+	if (stat(dest, &st) == 0) {
+		if (!S_ISDIR(st.st_mode)) {
+			fprintf(stderr, "the _data in volume %s is not directroy\n", dest);
+			return -1;
+		}
+
+		return 0;
+	}
+
+	if (errno != ENOENT) {
+		perror("access to volume failed\n");
+		return -1;
+	}
+
+	if (hyper_mkdir(dest, 0777) < 0) {
+		fprintf(stderr, "fail to create directroy %s\n", dest);
+		return -1;
+	}
+
+	return hyper_copy_dir(src, dest);
+}
+
+const char *INIT_VOLUME_FILENAME = ".hyper_file_volume_data_do_not_create_on_your_own";
+
+static int container_check_file_volume(char *hyper_path, const char **filename)
+{
+	struct dirent **list;
+	struct stat stbuf;
+	int i, num, found = 0;
+	char path[PATH_MAX];
+
+	*filename = NULL;
+	num = scandir(hyper_path, &list, NULL, NULL);
+	if (num < 0) {
+		/* No data in the volume yet, treat as non-file-volume */
+		if (errno == ENOENT) {
+			return 0;
+		}
+		perror("scan path failed");
+		return -1;
+	} else if (num != 3) {
+		fprintf(stdout, "%s has %d files/dirs\n", hyper_path, num - 2);
+		for (i = 0; i < num; i++) {
+			free(list[i]);
+		}
+		free(list);
+		return 0;
+	}
+
+	sprintf(path, "%s/%s", hyper_path, INIT_VOLUME_FILENAME);
+	for (i = 0; i < num; i++) {
+		if (strcmp(list[i]->d_name, ".") != 0 &&
+		    strcmp(list[i]->d_name, "..") != 0 &&
+		    strcmp(list[i]->d_name, INIT_VOLUME_FILENAME) == 0 &&
+		    stat(path, &stbuf) == 0 && S_ISREG(stbuf.st_mode)) {
+			found++;
+		}
+		free(list[i]);
+	}
+	free(list);
+
+	fprintf(stdout, "%s %s a file volume\n", hyper_path, found > 0?"is":"is not");
+	*filename = found > 0 ? INIT_VOLUME_FILENAME : NULL;
+	return 0;
+}
+
 static int container_setup_volume(struct hyper_container *container)
 {
 	int i;
@@ -26,140 +100,113 @@ static int container_setup_volume(struct hyper_container *container)
 	struct volume *vol;
 
 	for (i = 0; i < container->vols_num; i++) {
+		char volume[512];
+		char mountpoint[512];
+		char *options = NULL;
+		const char *filevolume = NULL;
 		vol = &container->vols[i];
 
 		if (vol->scsiaddr)
-			hyper_find_sd("/.oldroot", vol->scsiaddr, &vol->device);
+			hyper_find_sd(vol->scsiaddr, &vol->device);
 
-		sprintf(dev, "/.oldroot/dev/%s", vol->device);
+		sprintf(dev, "/dev/%s", vol->device);
 		sprintf(path, "/tmp/%s", vol->mountpoint);
+		sprintf(mountpoint, "./%s", vol->mountpoint);
+
 		fprintf(stdout, "mount %s to %s, tmp path %s\n",
 			dev, vol->mountpoint, path);
 
-		if (hyper_mkdir(path) < 0 || hyper_mkdir(vol->mountpoint) < 0) {
+		if (hyper_mkdir(path, 0755) < 0) {
 			perror("create volume dir failed");
 			return -1;
 		}
 
-		if (mount(dev, path, vol->fstype, 0, NULL) < 0) {
-			perror("mount volume device faled");
+		if (!strncmp(vol->fstype, "xfs", strlen("xfs")))
+			options = "nouuid";
+
+		if (mount(dev, path, vol->fstype, 0, options) < 0) {
+			perror("mount volume device failed");
 			return -1;
 		}
 
-		if (mount(path, vol->mountpoint, NULL, MS_BIND, NULL) < 0) {
-			perror("mount volume device faled");
+		sprintf(volume, "/%s/_data", path);
+		if (container_check_file_volume(volume, &filevolume) < 0)
+			return -1;
+
+		if (filevolume == NULL) {
+			if (hyper_mkdir(mountpoint, 0755) < 0) {
+				perror("create volume dir failed");
+				return -1;
+			}
+			if (vol->docker) {
+				if (container->initialize &&
+				    (container_populate_volume(mountpoint, volume) < 0)) {
+					fprintf(stderr, "fail to populate volume %s\n", mountpoint);
+					return -1;
+				}
+			} else if (hyper_mkdir(volume, 0777) < 0) {
+				/* First time mounting an empty volume */
+				perror("create _data dir failed");
+				return -1;
+			}
+		} else {
+			hyper_filize(mountpoint);
+			if (hyper_create_file(mountpoint) < 0) {
+				perror("create volume file failed");
+				return -1;
+			}
+			sprintf(volume, "/%s/_data/%s", path, filevolume);
+			/* 0777 so that any user can read/write the new file volume */
+			if (chmod(volume, 0777) < 0) {
+				fprintf(stderr, "fail to chmod directroy %s\n", volume);
+				return -1;
+			}
+		}
+
+		if (mount(volume, mountpoint, NULL, MS_BIND, NULL) < 0) {
+			perror("mount volume device failed");
 			return -1;
 		}
 
 		if (vol->readonly &&
-		    mount(path, vol->mountpoint, NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, NULL) < 0) {
-			perror("mount fsmap faled");
+		    mount(volume, mountpoint, NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, NULL) < 0) {
+			perror("mount fsmap failed");
 			return -1;
 		}
 
 		umount(path);
 	}
 
-	return 0;
-}
-
-static void container_unmount_oldroot(char *path)
-{
-	FILE *mtab;
-	struct mntent *mnt;
-	char *mntlist[128];
-	int i;
-	int n = 0;
-	char *filesys;
-
-	mtab = setmntent("/proc/mounts", "r");
-	if (mtab == NULL) {
-		fprintf(stderr, "cannot open /proc/mount");
-		return;
-	}
-
-	while (n < 128 && (mnt = getmntent(mtab))) {
-		if (strncmp(mnt->mnt_dir, path, strlen(path)))
-			continue;
-		mntlist[n++] = strdup(mnt->mnt_dir);
-	}
-
-	endmntent(mtab);
-
-	for (i = n - 1; i >= 0; i--) {
-		filesys = mntlist[i];
-		fprintf(stdout, "umount %s\n", filesys);
-		if (umount(mntlist[i]) < 0 && umount2(mntlist[i],
-			   MNT_DETACH) < 0) {
-			fprintf(stdout, "umount %s: %s failed\n",
-				filesys, strerror(errno));
-		}
-	}
-}
-
-static int container_setup_mount(struct hyper_container *container)
-{
-	int i, fd;
-	char src[512];
-	struct fsmap *map;
-
-	hyper_mkdir("/proc");
-	hyper_mkdir("/sys");
-	hyper_mkdir("/dev");
-
-	if (mount("proc", "/proc", "proc", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) < 0 ||
-	    mount("sysfs", "/sys", "sysfs", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) < 0 ||
-	    mount("devtmpfs", "/dev", "devtmpfs", MS_NOSUID, NULL) < 0) {
-		perror("mount basic filesystem for container failed");
-		return -1;
-	}
-
-	if (hyper_mkdir("/dev/shm") < 0) {
-		fprintf(stderr, "create /dev/shm failed\n");
-		return -1;
-	}
-
-	if (mount("tmpfs", "/dev/shm/", "tmpfs", MS_NOSUID| MS_NODEV, NULL) < 0) {
-		perror("mount shm failed");
-		return -1;
-	}
-
-	if (hyper_mkdir("/dev/pts") < 0) {
-		fprintf(stderr, "create /dev/pts failed\n");
-		return -1;
-	}
-
-	if (sprintf(src, "/.oldroot/tmp/hyper/%s/devpts", container->id) < 0) {
-		fprintf(stderr, "get container devpts failed\n");
-		return -1;
-	}
-
-	if (mount(src, "/dev/pts/", NULL, MS_BIND, NULL) < 0) {
-		perror("move pts to /dev/pts failed");
-		return -1;
-	}
-
-	if (unlink("/dev/ptmx") < 0)
-		perror("remove /dev/ptmx failed");
-	if (symlink("/dev/pts/ptmx", "/dev/ptmx") < 0)
-		perror("link /dev/pts/ptmx to /dev/ptmx failed");
-
 	for (i = 0; i < container->maps_num; i++) {
 		struct stat st;
+		char *src, path[512], volume[512];
+		struct fsmap *map = &container->maps[i];
+		char mountpoint[512];
 
-		map = &container->maps[i];
-
-		sprintf(src, "/.oldroot/tmp/hyper/shared/%s", map->source);
-		fprintf(stdout, "mount %s to %s\n", src, map->path);
+		sprintf(path, "/tmp/hyper/shared/%s", map->source);
+		sprintf(mountpoint, "./%s", map->path);
+		fprintf(stdout, "mount %s to %s\n", path, mountpoint);
 
+		src = path;
 		stat(src, &st);
+
 		if (st.st_mode & S_IFDIR) {
-			if (hyper_mkdir(map->path) < 0) {
+			if (hyper_mkdir(mountpoint, 0755) < 0) {
 				perror("create map dir failed");
 				continue;
 			}
+			if (map->docker) {
+				/* converted from volume */
+				sprintf(volume, "%s/_data", path);
+				src = volume;
+				if (container->initialize &&
+				    (container_populate_volume(mountpoint, volume) < 0)) {
+					fprintf(stderr, "fail to populate volume %s\n", mountpoint);
+					continue;
+				}
+			}
 		} else {
-			fd = open(map->path, O_CREAT|O_WRONLY, 0755);
+			int fd = open(mountpoint, O_CREAT|O_WRONLY, 0755);
 			if (fd < 0) {
 				perror("create map file failed");
 				continue;
@@ -167,65 +214,201 @@ static int container_setup_mount(struct hyper_container *container)
 			close(fd);
 		}
 
-		if (mount(src, map->path, NULL, MS_BIND, NULL) < 0) {
-			perror("mount fsmap faled");
+		if (mount(src, mountpoint, NULL, MS_BIND, NULL) < 0) {
+			perror("mount fsmap failed");
 			continue;
 		}
 
 		if (map->readonly == 0)
 			continue;
 
-		if (mount(src, map->path, NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, NULL) < 0)
-			perror("mount fsmap faled");
+		if (mount(src, mountpoint, NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, NULL) < 0)
+			perror("mount fsmap failed");
 	}
 
 	return 0;
 }
 
+static int container_setup_modules(struct hyper_container *container)
+{
+	struct stat st;
+	struct utsname uts;
+	char src[512], dst[512];
+
+	if (uname(&uts) < 0) {
+		perror("fail to call uname");
+		return -1;
+	}
+
+	sprintf(src, "/lib/modules/%s", uts.release);
+	sprintf(dst, "./%s", src);
+
+	if (stat(dst, &st) == 0) {
+		struct dirent **list;
+		int num;
+
+		if (!S_ISDIR(st.st_mode)) {
+			return -1;
+		}
+
+		num = scandir(dst, &list, NULL, NULL);
+		if (num > 2) {
+			fprintf(stdout, "%s is not null, %d", dst, num);
+			return 0;
+		}
+	} else if (errno == ENOENT) {
+		if (hyper_mkdir(dst, 0755) < 0)
+			return -1;
+	} else {
+		return -1;
+	}
+
+	if (mount(src, dst, NULL, MS_BIND, NULL) < 0) {
+		perror("mount bind modules failed");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int container_setup_mount(struct hyper_container *container)
+{
+	char src[512];
+
+	// current dir is container rootfs, the operations on "./PATH" are the operations on container's "/PATH"
+	hyper_mkdir("./proc", 0755);
+	hyper_mkdir("./sys", 0755);
+	hyper_mkdir("./dev", 0755);
+	hyper_mkdir("./lib/modules", 0755);
+
+	if (mount("proc", "./proc", "proc", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) < 0 ||
+	    mount("sysfs", "./sys", "sysfs", MS_NOSUID| MS_NODEV| MS_NOEXEC, NULL) < 0 ||
+	    mount("devtmpfs", "./dev", "devtmpfs", MS_NOSUID, NULL) < 0) {
+		perror("mount basic filesystem for container failed");
+		return -1;
+	}
+
+	if (hyper_mkdir("./dev/shm", 0755) < 0) {
+		fprintf(stderr, "create /dev/shm failed\n");
+		return -1;
+	}
+
+	if (mount("tmpfs", "./dev/shm/", "tmpfs", MS_NOSUID| MS_NODEV, NULL) < 0) {
+		perror("mount shm failed");
+		return -1;
+	}
+
+	if (hyper_mkdir("./dev/pts", 0755) < 0) {
+		fprintf(stderr, "create /dev/pts failed\n");
+		return -1;
+	}
+
+	if (sprintf(src, "/tmp/hyper/%s/devpts", container->id) < 0) {
+		fprintf(stderr, "get container devpts failed\n");
+		return -1;
+	}
+
+	if (mount(src, "./dev/pts/", NULL, MS_BIND, NULL) < 0) {
+		perror("move pts to /dev/pts failed");
+		return -1;
+	}
+
+	if (unlink("./dev/ptmx") < 0)
+		perror("remove /dev/ptmx failed");
+	if (symlink("/dev/pts/ptmx", "./dev/ptmx") < 0)
+		perror("link /dev/pts/ptmx to /dev/ptmx failed");
+
+	symlink("/proc/self/fd", "./dev/fd");
+	symlink("/proc/self/fd/0", "./dev/stdin");
+	symlink("/proc/self/fd/1", "./dev/stdout");
+	symlink("/proc/self/fd/2", "./dev/stderr");
+
+	return 0;
+}
+
+static int container_recreate_file(char *filename)
+{
+	struct stat stbuf;
+
+	fprintf(stdout, "recreate file %s\n", filename);
+	if (stat(filename, &stbuf) < 0) {
+		if (errno != ENOENT) {
+			fprintf(stderr, "failed to stat %s: %d\n", filename, errno);
+			return -1;
+		}
+		return hyper_create(filename);
+	}
+	if (stbuf.st_mode & S_IFREG && stbuf.st_size == 0)
+		return 0;
+
+	hyper_unlink(filename);
+	return hyper_create(filename);
+}
+
+static int container_recreate_symlink(char *oldpath, char *newpath)
+{
+	fprintf(stdout, "recreate symlink %s to %s\n", newpath, oldpath);
+	hyper_unlink(newpath);
+	return hyper_symlink(oldpath, newpath);
+}
+
+/*
+ * Docker uses the init layer to protect against unwanted side effects on
+ * the rw layer. We recreate the same files here to have similar effect.
+ * Docker also creates directories like /dev/pts, /dev/shm, /proc, /sys,
+ * which we have over ridden in container_setup_mount, so no need to create
+ * them here.
+ */
+static int container_setup_init_layer(struct hyper_container *container,
+				      int setup_dns)
+{
+	if (!container->initialize)
+		return 0;
+
+	hyper_mkdir("./etc/", 0755);
+
+	if (setup_dns && container_recreate_file("./etc/resolv.conf") < 0)
+		return -1;
+
+	if (container_recreate_file("./etc/hosts") < 0)
+		return -1;
+
+	if (container_recreate_file("./etc/hostname") < 0)
+		return -1;
+
+	if (container_recreate_symlink("/proc/mounts", "./etc/mtab") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int container_setup_sysctl(struct hyper_container *container)
 {
-	int i, size, len, l, fd;
+	int i;
 	struct sysctl *sys;
 
 	for (i = 0; i < container->sys_num; i++) {
 		char path[256];
 
-		len = 0;
 		sys = &container->sys[i];
-		size = strlen(sys->value);
 
 		sprintf(path, "/proc/sys/%s", sys->path);
 		fprintf(stdout, "sysctl %s value %s\n", sys->path, sys->value);
 
-		fd = open(path, O_WRONLY);
-		if (fd < 0) {
-			perror("open file failed");
-			goto out;
+		if (hyper_write_file(path, sys->value, strlen(sys->value)) < 0) {
+			fprintf(stderr, "sysctl: write %s to %s failed\n", sys->value, path);
+			return -1;
 		}
-
-		while (len < size) {
-			l = write(fd, sys->value + len, size - len);
-			if (l < 0) {
-				perror("fail to write sysctl");
-				close(fd);
-				goto out;
-			}
-			len += l;
-		}
-
-		close(fd);
 	}
 
 	return 0;
-out:
-	return -1;
 }
 
 static int container_setup_dns(struct hyper_container *container)
 {
 	int fd;
 	struct stat st;
-	char *src = "/.oldroot/tmp/hyper/resolv.conf";
+	char *src = "/tmp/hyper/resolv.conf";
 
 	if (stat(src, &st) < 0) {
 		if (errno == ENOENT) {
@@ -237,16 +420,16 @@ static int container_setup_dns(struct hyper_container *container)
 		return -1;
 	}
 
-	hyper_mkdir("/etc");
+	hyper_mkdir("./etc", 0755);
 
-	fd = open("/etc/resolv.conf", O_CREAT| O_WRONLY, 0644);
+	fd = open("./etc/resolv.conf", O_CREAT| O_WRONLY, 0644);
 	if (fd < 0) {
 		perror("create /etc/resolv.conf failed");
 		return -1;
 	}
 	close(fd);
 
-	if (mount(src, "/etc/resolv.conf", NULL, MS_BIND, NULL) < 0) {
+	if (mount(src, "./etc/resolv.conf", NULL, MS_BIND, NULL) < 0) {
 		perror("bind to /etc/resolv.conf failed");
 		return -1;
 	}
@@ -256,19 +439,14 @@ static int container_setup_dns(struct hyper_container *container)
 
 static int container_setup_workdir(struct hyper_container *container)
 {
-	if (container->workdir && chdir(container->workdir) < 0) {
-		perror("change work directory failed");
-		return -1;
+	if (container->initialize) {
+		// create workdir
+		return hyper_mkdir(container->exec.workdir, 0755);
 	}
 
 	return 0;
 }
 
-static int container_setup_tty(int fd, struct hyper_container *container)
-{
-	return hyper_dup_exec_tty(fd, &container->exec);
-}
-
 static int hyper_rescan_scsi(void)
 {
 	struct dirent **list;
@@ -316,30 +494,21 @@ static int hyper_rescan_scsi(void)
 
 struct hyper_container_arg {
 	struct hyper_container	*c;
-	int			ipcns;
-	int			utsns;
+	struct hyper_pod	*pod;
 	int			pipe[2];
+	int			pipens[2];
 };
 
-static int hyper_container_init(void *data)
+static int hyper_setup_container_rootfs(void *data)
 {
 	struct hyper_container_arg *arg = data;
 	struct hyper_container *container = arg->c;
-	char root[512], oldroot[512];
+	char root[512], rootfs[512];
+	int setup_dns;
+	uint32_t type;
 
-	fprintf(stdout, "%s in\n", __func__);
-	if (container->exec.argv == NULL) {
-		fprintf(stdout, "no cmd!\n");
-		goto fail;
-	}
-
-	if (setns(arg->ipcns, CLONE_NEWIPC) < 0) {
-		perror("setns to ipcns of pod init faild");
-		goto fail;
-	}
-
-	if (setns(arg->utsns, CLONE_NEWUTS) < 0) {
-		perror("setns to ipcns of pod init faild");
+	if (hyper_enter_sandbox(arg->pod, -1) < 0) {
+		perror("enter sandbox failed");
 		goto fail;
 	}
 
@@ -348,11 +517,6 @@ static int hyper_container_init(void *data)
 		goto fail;
 	}
 
-	if (hyper_setup_env(container->envs, container->envs_num) < 0) {
-		fprintf(stdout, "setup env failed\n");
-		goto fail;
-	}
-
 	if (mount("", "/", NULL, MS_SLAVE|MS_REC, NULL) < 0) {
 		perror("mount SLAVE failed");
 		goto fail;
@@ -364,7 +528,7 @@ static int hyper_container_init(void *data)
 	}
 
 	sprintf(root, "/tmp/hyper/%s/root/", container->id);
-	if (hyper_mkdir(root) < 0) {
+	if (hyper_mkdir(root, 0755) < 0) {
 		perror("make root directroy failed");
 		goto fail;
 	}
@@ -373,14 +537,21 @@ static int hyper_container_init(void *data)
 
 	if (container->fstype) {
 		char dev[128];
+		char *options = NULL;
 
-		if (container->scsiaddr)
-			hyper_find_sd("", container->scsiaddr, &container->image);
+		if (container->scsiaddr) {
+			free(container->image);
+			container->image = NULL;
+			hyper_find_sd(container->scsiaddr, &container->image);
+		}
 
 		sprintf(dev, "/dev/%s", container->image);
 		fprintf(stdout, "device %s\n", dev);
 
-		if (mount(dev, root, container->fstype, 0, NULL) < 0) {
+		if (!strncmp(container->fstype, "xfs", strlen("xfs")))
+			options = "nouuid";
+
+		if (mount(dev, root, container->fstype, 0, options) < 0) {
 			perror("mount device failed");
 			goto fail;
 		}
@@ -399,26 +570,20 @@ static int hyper_container_init(void *data)
 	fprintf(stdout, "root directory for container is %s/%s, init task %s\n",
 		root, container->rootfs, container->exec.argv[0]);
 
-	sprintf(oldroot, "%s/%s/.oldroot", root, container->rootfs);
-	if (hyper_mkdir(oldroot) < 0) {
-		perror("make oldroot directroy failed");
+	sprintf(rootfs, "%s/%s/", root, container->rootfs);
+	if (mount(rootfs, rootfs, NULL, MS_BIND|MS_REC, NULL) < 0) {
+		perror("failed to bind rootfs");
 		goto fail;
 	}
+	chdir(rootfs);
 
-	if (mount("/", oldroot, NULL, MS_BIND|MS_REC, NULL) < 0) {
-		perror("bind oldroot failed");
-		goto fail;
-	}
-	/* reuse oldroot array */
-	sprintf(oldroot, "%s/%s/", root, container->rootfs);
-	/* pivot_root won't work, see
-	 * Documention/filesystem/ramfs-rootfs-initramfs.txt */
-	chroot(oldroot);
-
-	chdir("/");
-
-	if (container_setup_volume(container) < 0) {
-		fprintf(stderr, "container sets up voulme failed\n");
+	/*
+	 * Recreate dns resolver iif configured by pod spec. Other cases
+	 * are handled by hyperd instead.
+	 */
+	setup_dns = arg->pod->dns != NULL && arg->pod->d_num > 0;
+	if (container_setup_init_layer(container, setup_dns) < 0) {
+		fprintf(stderr, "container sets up init layer failed\n");
 		goto fail;
 	}
 
@@ -427,8 +592,11 @@ static int hyper_container_init(void *data)
 		goto fail;
 	}
 
-	if (container_setup_sysctl(container) < 0) {
-		fprintf(stderr, "container sets up sysctl failed\n");
+	// ignore error of setup modules
+	container_setup_modules(container);
+
+	if (container_setup_volume(container) < 0) {
+		fprintf(stderr, "container sets up voulme failed\n");
 		goto fail;
 	}
 
@@ -437,33 +605,40 @@ static int hyper_container_init(void *data)
 		goto fail;
 	}
 
+	// manipulate the rootfs of the container/namespace: move the prepared path @rootfs to /
+	if (mount(rootfs, "/", NULL, MS_MOVE, NULL) < 0) {
+		perror("failed to move rootfs");
+		goto fail;
+	}
+	/* pivot_root won't work, see
+	 * Documention/filesystem/ramfs-rootfs-initramfs.txt */
+	chroot(".");
+
+	chdir("/");
+
+	if (container_setup_sysctl(container) < 0) {
+		fprintf(stderr, "container sets up sysctl failed\n");
+		goto fail;
+	}
+
 	if (container_setup_workdir(container) < 0) {
 		fprintf(stderr, "container sets up work directory failed\n");
 		goto fail;
 	}
 
-	container_unmount_oldroot("/.oldroot");
-
-	fflush(stdout);
-
-	if (container_setup_tty(arg->pipe[1], container) < 0) {
-		fprintf(stdout, "setup tty failed\n");
+	/* wait for ns-opened ready message */
+	if (hyper_get_type(arg->pipens[0], &type) < 0 || type != READY) {
+		fprintf(stderr, "wait for /proc/self/ns/mnt opened failed\n");
 		goto fail;
 	}
 
-	symlink("/proc/self/fd", "/dev/fd");
-	symlink("/proc/self/fd/0", "/dev/stdin");
-	symlink("/proc/self/fd/1", "/dev/stdout");
-	symlink("/proc/self/fd/2", "/dev/stderr");
-
-	execvp(container->exec.argv[0], container->exec.argv);
-	perror("exec container command failed");
-
-	_exit(-1);
+	hyper_send_type(arg->pipe[1], READY);
+	fflush(NULL);
+	_exit(0);
 
 fail:
 	hyper_send_type(arg->pipe[1], ERROR);
-	_exit(-1);
+	_exit(125);
 }
 
 static int hyper_setup_pty(struct hyper_container *c)
@@ -472,7 +647,7 @@ static int hyper_setup_pty(struct hyper_container *c)
 
 	sprintf(root, "/tmp/hyper/%s/devpts/", c->id);
 
-	if (hyper_mkdir(root) < 0) {
+	if (hyper_mkdir(root, 0755) < 0) {
 		perror("make container pts directroy failed");
 		return -1;
 	}
@@ -483,33 +658,31 @@ static int hyper_setup_pty(struct hyper_container *c)
 		return -1;
 	}
 
-	if (hyper_setup_exec_tty(&c->exec) < 0) {
-		fprintf(stderr, "setup container pts failed\n");
-		return -1;
-	}
-
 	return 0;
 }
 
-int hyper_start_container(struct hyper_container *container,
-			  int utsns, int ipcns, struct hyper_pod *pod)
+int hyper_setup_container(struct hyper_container *container, struct hyper_pod *pod)
 {
-	int stacksize = getpagesize() * 4;
+	int stacksize = getpagesize() * 42;
 	struct hyper_container_arg arg = {
 		.c	= container,
-		.utsns	= utsns,
-		.ipcns	= ipcns,
+		.pod	= pod,
 		.pipe	= {-1, -1},
+		.pipens = {-1, -1},
 	};
 	int flags = CLONE_NEWNS | SIGCHLD;
 	char path[128];
-	uint32_t type;
 	void *stack;
+	uint32_t type;
 	int pid;
 
-	if (container->image == NULL || container->exec.argv == NULL) {
-		fprintf(stdout, "container root image %s, argv %p\n",
-			container->image, container->exec.argv);
+	if (pipe2(arg.pipe, O_CLOEXEC) < 0 || pipe2(arg.pipens, O_CLOEXEC) < 0) {
+		perror("create pipe between pod init execcmd failed");
+		goto fail;
+	}
+
+	if (hyper_setup_container_portmapping(container, pod) < 0) {
+		perror("fail to setup port mapping for container");
 		goto fail;
 	}
 
@@ -518,66 +691,49 @@ int hyper_start_container(struct hyper_container *container,
 		goto fail;
 	}
 
-	if (pipe2(arg.pipe, O_CLOEXEC) < 0) {
-		perror("create pipe between pod init execcmd failed");
-		goto fail;
-	}
-
-	if (hyper_watch_exec_pty(&container->exec, pod) < 0) {
-		fprintf(stderr, "faile to watch container pty\n");
-		goto fail;
-	}
-
 	stack = malloc(stacksize);
 	if (stack == NULL) {
 		perror("fail to allocate stack for container init");
 		goto fail;
 	}
 
-	pid = clone(hyper_container_init, stack + stacksize, flags, &arg);
+	pid = clone(hyper_setup_container_rootfs, stack + stacksize, flags, &arg);
 	free(stack);
 	if (pid < 0) {
 		perror("create child process failed");
 		goto fail;
 	}
-	sprintf(path, "/proc/%d/ns/mnt", pid);
 
+	sprintf(path, "/proc/%d/ns/mnt", pid);
 	container->ns = open(path, O_RDONLY | O_CLOEXEC);
 	if (container->ns < 0) {
 		perror("open container mount ns failed");
 		goto fail;
 	}
+	hyper_send_type(arg.pipens[1], READY);
 
 	/* wait for ready message */
 	if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) {
-		fprintf(stderr, "wait for container started failed\n");
+		fprintf(stderr, "wait for setup container rootfs failed\n");
 		goto fail;
 	}
 
-	container->exec.pid = pid;
-	list_add_tail(&container->exec.list, &pod->exec_head);
-	container->exec.ref++;
-
 	close(arg.pipe[0]);
 	close(arg.pipe[1]);
-
-	fprintf(stdout, "container %s,init pid %d,ref %d\n", container->id, pid, container->exec.ref);
+	close(arg.pipens[0]);
+	close(arg.pipens[1]);
 	return 0;
 fail:
+	close(container->ns);
+	container->ns = -1;
 	close(arg.pipe[0]);
 	close(arg.pipe[1]);
-	close(container->ns);
-	hyper_reset_event(&container->exec.e);
-	hyper_reset_event(&container->exec.errev);
-	container->ns = -1;
-	fprintf(stdout, "container %s init exit code %d\n", container->id, -1);
-	container->exec.code = -1;
-	container->exec.seq = 0;
-	container->exec.ref = 0;
+	close(arg.pipens[0]);
+	close(arg.pipens[1]);
 	return -1;
 }
 
-struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id)
+struct hyper_container *hyper_find_container(struct hyper_pod *pod, const char *id)
 {
 	struct hyper_container *c;
 
@@ -594,7 +750,7 @@ struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id)
 	return NULL;
 }
 
-void hyper_cleanup_container(struct hyper_container *c)
+void hyper_cleanup_container(struct hyper_container *c, struct hyper_pod *pod)
 {
 	char root[512];
 
@@ -603,6 +759,7 @@ void hyper_cleanup_container(struct hyper_container *c)
 		perror("umount devpts failed");
 
 	close(c->ns);
+	hyper_cleanup_container_portmapping(c, pod);
 	hyper_free_container(c);
 }
 
@@ -611,7 +768,7 @@ void hyper_cleanup_containers(struct hyper_pod *pod)
 	struct hyper_container *c, *n;
 
 	list_for_each_entry_safe(c, n, &pod->containers, list)
-		hyper_cleanup_container(c);
+		hyper_cleanup_container(c, pod);
 
 	pod->remains = 0;
 }

src/container.h

@@ -3,23 +3,20 @@
 
 #include "exec.h"
 
-struct env {
-	char	*env;
-	char	*value;
-};
-
 struct volume {
 	char	*device;
 	char	*scsiaddr;
 	char	*mountpoint;
 	char	*fstype;
 	int	readonly;
+	int	docker;
 };
 
 struct fsmap {
 	char	*source;
 	char	*path;
 	int	readonly;
+	int	docker;
 };
 
 struct sysctl {
@@ -27,33 +24,40 @@ struct sysctl {
 	char	*value;
 };
 
+struct port {
+	int  host_port;
+	int  container_port;
+	char *protocol;
+};
+
 struct hyper_container {
+	struct list_head	list;
+	struct hyper_exec	exec;
+	int			ns;
+	uint32_t		code;
+
+	// configs
 	char			*id;
 	char			*rootfs;
 	char			*image;
 	char			*scsiaddr;
-	char			*workdir;
 	char			*fstype;
 	struct volume		*vols;
-	struct env		*envs;
 	struct fsmap		*maps;
 	struct sysctl		*sys;
+	struct port		*ports;
 	int			vols_num;
-	int			envs_num;
 	int			maps_num;
 	int			sys_num;
-	int			ns;
-	uint32_t		code;
-	struct list_head	list;
-	struct hyper_exec	exec;
+	int			ports_num;
+	int			initialize;
 };
 
 struct hyper_pod;
 
-int hyper_start_container(struct hyper_container *container,
-			  int utsns, int ipcns, struct hyper_pod *pod);
-struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id);
-void hyper_cleanup_container(struct hyper_container *container);
+int hyper_setup_container(struct hyper_container *container, struct hyper_pod *pod);
+struct hyper_container *hyper_find_container(struct hyper_pod *pod, const char *id);
+void hyper_cleanup_container(struct hyper_container *container, struct hyper_pod *pod);
 void hyper_cleanup_containers(struct hyper_pod *pod);
 void hyper_free_container(struct hyper_container *c);
 

src/event.c

@@ -10,25 +10,25 @@
 #include "hyper.h"
 #include "event.h"
 
-void hyper_reset_event(struct hyper_event *de)
+void hyper_reset_event(struct hyper_event *he)
 {
-	free(de->rbuf.data);
-	free(de->wbuf.data);
-	close(de->fd);
-	memset(de, 0, sizeof(*de));
-	de->fd = -1;
+	free(he->rbuf.data);
+	free(he->wbuf.data);
+	close(he->fd);
+	memset(he, 0, sizeof(*he));
+	he->fd = -1;
 }
 
-int hyper_init_event(struct hyper_event *de, struct hyper_event_ops *ops, void *arg)
+int hyper_init_event(struct hyper_event *he, struct hyper_event_ops *ops, void *arg)
 {
-	struct hyper_buf *rbuf = &de->rbuf;
-	struct hyper_buf *wbuf = &de->wbuf;
+	struct hyper_buf *rbuf = &he->rbuf;
+	struct hyper_buf *wbuf = &he->wbuf;
 
 	memset(rbuf, 0, sizeof(*rbuf));
 	memset(wbuf, 0, sizeof(*wbuf));
 
-	de->ops		= ops;
-	de->ptr		= arg;
+	he->ops		= ops;
+	he->ptr		= arg;
 	rbuf->size	= ops->rbuf_size;
 	wbuf->size	= ops->wbuf_size;
 
@@ -51,22 +51,22 @@ int hyper_init_event(struct hyper_event *de, struct hyper_event_ops *ops, void *
 	return 0;
 }
 
-int hyper_add_event(int efd, struct hyper_event *de, int flag)
+int hyper_add_event(int efd, struct hyper_event *he, int flag)
 {
 	struct epoll_event event = {
 		.events		= flag,
-		.data.ptr	= de,
+		.data.ptr	= he,
 	};
 
-	de->flag = flag;
-	if (hyper_setfd_nonblock(de->fd) < 0) {
+	he->flag = flag;
+	if (hyper_setfd_nonblock(he->fd) < 0) {
 		perror("set fd nonblock failed");
 		return -1;
 	}
 
-	fprintf(stdout, "%s add event fd %d, %p\n", __func__, de->fd, de->ops);
+	fprintf(stdout, "%s add event fd %d, %p\n", __func__, he->fd, he->ops);
 
-	if (epoll_ctl(efd, EPOLL_CTL_ADD, de->fd, &event) < 0) {
+	if (epoll_ctl(efd, EPOLL_CTL_ADD, he->fd, &event) < 0) {
 		perror("epoll_ctl fd failed");
 		return -1;
 	}
@@ -74,21 +74,21 @@ int hyper_add_event(int efd, struct hyper_event *de, int flag)
 	return 0;
 }
 
-int hyper_modify_event(int efd, struct hyper_event *de, int flag)
+int hyper_modify_event(int efd, struct hyper_event *he, int flag)
 {
 	struct epoll_event event = {
 		.events		= flag,
-		.data.ptr	= de,
+		.data.ptr	= he,
 	};
 
-	if (de->flag == flag)
+	if (he->flag == flag)
 		return 0;
 
-	de->flag = flag;
+	he->flag = flag;
 	fprintf(stdout, "%s modify event fd %d, %p, event %d\n",
-			__func__, de->fd, de, flag);
+			__func__, he->fd, he, flag);
 
-	if (epoll_ctl(efd, EPOLL_CTL_MOD, de->fd, &event) < 0) {
+	if (epoll_ctl(efd, EPOLL_CTL_MOD, he->fd, &event) < 0) {
 		perror("epoll_ctl fd failed");
 		return -1;
 	}
@@ -96,39 +96,59 @@ int hyper_modify_event(int efd, struct hyper_event *de, int flag)
 	return 0;
 }
 
-static int hyper_getmsg_len(struct hyper_event *de, uint32_t *len)
+int hyper_requeue_event(int efd, struct hyper_event *ev)
 {
-	struct hyper_buf *buf = &de->rbuf;
+	struct epoll_event event = {
+		.events		= ev->flag,
+		.data.ptr	= ev,
+	};
 
-	if (buf->get < de->ops->len_offset + 4)
+	if (epoll_ctl(efd, EPOLL_CTL_DEL, ev->fd, NULL) < 0) {
+		perror("epoll_ctl del fd failed");
 		return -1;
+	}
+
+	if (epoll_ctl(efd, EPOLL_CTL_ADD, ev->fd, &event) < 0) {
+		perror("epoll_ctl add fd failed");
+		return -1;
+	}
 
-	*len = hyper_get_be32(buf->data + de->ops->len_offset);
 	return 0;
 }
 
-int hyper_event_read(struct hyper_event *de)
+static int hyper_getmsg_len(struct hyper_event *he, uint32_t *len)
 {
-	struct hyper_buf *buf = &de->rbuf;
+	struct hyper_buf *buf = &he->rbuf;
+
+	if (buf->get < he->ops->len_offset + 4)
+		return -1;
+
+	*len = hyper_get_be32(buf->data + he->ops->len_offset);
+	return 0;
+}
+
+int hyper_event_read(struct hyper_event *he, int efd)
+{
+	struct hyper_buf *buf = &he->rbuf;
 	uint32_t len = 4;
 	uint8_t data[4];
-	int offset = de->ops->len_offset;
+	int offset = he->ops->len_offset;
 	int end = offset + 4;
 	int size;
 
 	fprintf(stdout, "%s\n", __func__);
 
-	while (hyper_getmsg_len(de, &len) < 0) {
-		size = read(de->fd, buf->data + buf->get, end - buf->get);
+	while (hyper_getmsg_len(he, &len) < 0) {
+		size = read(he->fd, buf->data + buf->get, end - buf->get);
 		if (size > 0) {
 			buf->get += size;
 			fprintf(stdout, "already read %" PRIu32 " bytes data\n",
 				buf->get);
 
-			if (de->ops->ack) {
+			if (he->ops->ack) {
 				/* control channel, need ack */
 				hyper_set_be32(data, size);
-				hyper_send_msg(de->fd, NEXT, 4, data);
+				hyper_send_msg(he->fd, NEXT, 4, data);
 			}
 			continue;
 		}
@@ -151,15 +171,15 @@ int hyper_event_read(struct hyper_event *de)
 	}
 
 	while (buf->get < len) {
-		size = read(de->fd, buf->data + buf->get, len - buf->get);
+		size = read(he->fd, buf->data + buf->get, len - buf->get);
 		if (size > 0) {
 			buf->get += size;
 			fprintf(stdout, "read %d bytes data, total data %" PRIu32 "\n",
 				size, buf->get);
-			if (de->ops->ack) {
+			if (he->ops->ack) {
 				/* control channel, need ack */
 				hyper_set_be32(data, size);
-				hyper_send_msg(de->fd, NEXT, 4, data);
+				hyper_send_msg(he->fd, NEXT, 4, data);
 			}
 
 			continue;
@@ -178,7 +198,7 @@ int hyper_event_read(struct hyper_event *de)
 	}
 
 	/* get the whole data */
-	if (de->ops->handle(de, len) != 0)
+	if (he->ops->handle(he, len) != 0)
 		return -1;
 
 	/* len: length of the already get new data */
@@ -188,14 +208,14 @@ int hyper_event_read(struct hyper_event *de)
 	return 0;
 }
 
-int hyper_event_write(struct hyper_event *de)
+int hyper_event_write(struct hyper_event *he, int efd)
 {
-	struct hyper_buf *buf = &de->wbuf;
+	struct hyper_buf *buf = &he->wbuf;
 	uint32_t len = 0;
 	int size = 0;
 
 	while (len < buf->get) {
-		size = write(de->fd, buf->data + len, buf->get - len);
+		size = write(he->fd, buf->data + len, buf->get - len);
 		if (size <= 0) {
 			if (errno == EINTR)
 				continue;
@@ -210,49 +230,53 @@ int hyper_event_write(struct hyper_event *de)
 	memmove(buf->data, buf->data + len, buf->get);
 
 	if (buf->get == 0) {
-		hyper_modify_event(ctl.efd, de, EPOLLIN);
+		hyper_modify_event(ctl.efd, he, he->flag & ~(EPOLLOUT| EPOLLPRI));
+	} else if (!FULL(buf)) {
+		hyper_modify_event(ctl.efd, he, he->flag & ~EPOLLPRI);
 	}
 
 	return 0;
 }
 
-void hyper_event_hup(struct hyper_event *de, int efd)
+void hyper_event_hup(struct hyper_event *he, int efd)
 {
-	if (epoll_ctl(efd, EPOLL_CTL_DEL, de->fd, NULL) < 0)
+	if (epoll_ctl(efd, EPOLL_CTL_DEL, he->fd, NULL) < 0)
 		perror("epoll_ctl del epoll event failed");
-	hyper_reset_event(de);
+	hyper_reset_event(he);
 }
 
 int hyper_handle_event(int efd, struct epoll_event *event)
 {
-	struct hyper_event *de = event->data.ptr;
-	fprintf(stdout, "%s get event %d, de %p, fd %d. ops %p\n",
-			__func__, event->events, de, de->fd, de->ops);
+	struct hyper_event *he = event->data.ptr;
+	fprintf(stdout, "%s get event %d, he %p, fd %d. ops %p\n",
+			__func__, event->events, he, he->fd, he->ops);
 
 	/* do not handle hup event if have in event */
-	if (event->events & EPOLLIN) {
-		fprintf(stdout, "%s event EPOLLIN, de %p, fd %d, %p\n",
-			__func__, de, de->fd, de->ops);
-		if (de->ops->read(de) < 0)
+	if ((event->events & EPOLLIN) && he->ops->read) {
+		fprintf(stdout, "%s event EPOLLIN, he %p, fd %d, %p\n",
+			__func__, he, he->fd, he->ops);
+		if (he->ops->read && he->ops->read(he, efd) < 0)
 			return -1;
 	} else if (event->events & EPOLLHUP) {
-		fprintf(stdout, "%s event EPOLLHUP, de %p, fd %d, %p\n",
-			__func__, de, de->fd, de->ops);
-		if (de->ops->hup)
-			de->ops->hup(de, efd);
+		fprintf(stdout, "%s event EPOLLHUP, he %p, fd %d, %p\n",
+			__func__, he, he->fd, he->ops);
+		if (he->ops->hup)
+			he->ops->hup(he, efd);
 		return 0;
 	}
 
 	if (event->events & EPOLLOUT) {
-		fprintf(stdout, "%s event EPOLLOUT, de %p, fd %d, %p\n",
-			__func__, de, de->fd, de->ops);
-		if (de->ops->write && de->ops->write(de) < 0)
+		fprintf(stdout, "%s event EPOLLOUT, he %p, fd %d, %p\n",
+			__func__, he, he->fd, he->ops);
+		if (he->ops->write && he->ops->write(he, efd) < 0)
 			return -1;
 	}
 
 	if (event->events & EPOLLERR) {
 		fprintf(stderr, "get epoll err of not epool in event\n");
-		return -1;
+		if (he->ops->hup)
+			he->ops->hup(he, efd);
+		return 0;
 	}
 
 	return 0;

src/event.h

@@ -7,8 +7,8 @@
 struct hyper_event;
 
 struct hyper_event_ops {
-	int		(*read)(struct hyper_event *e);
-	int		(*write)(struct hyper_event *e);
+	int		(*read)(struct hyper_event *e, int efd);
+	int		(*write)(struct hyper_event *e, int efd);
 	int		(*handle)(struct hyper_event *e, uint32_t len);
 	void		(*hup)(struct hyper_event *e, int efd);
 	int		rbuf_size;
@@ -32,13 +32,17 @@ struct hyper_event {
 	void			*ptr;
 };
 
+#define FULL(buf) \
+	(buf->size - buf->get <= 12)
+
 int hyper_add_event(int efd, struct hyper_event *de, int flag);
 int hyper_modify_event(int efd, struct hyper_event *de, int flag);
+int hyper_requeue_event(int efd, struct hyper_event *ev);
 int hyper_init_event(struct hyper_event *de, struct hyper_event_ops *ops,
 		     void *arg);
 int hyper_handle_event(int efd, struct epoll_event *event);
 void hyper_reset_event(struct hyper_event *de);
 void hyper_event_hup(struct hyper_event *de, int efd);
-int hyper_event_read(struct hyper_event *de);
-int hyper_event_write(struct hyper_event *de);
+int hyper_event_read(struct hyper_event *dei, int efd);
+int hyper_event_write(struct hyper_event *de, int efd);
 #endif

src/exec.h

@@ -4,37 +4,51 @@
 #include "list.h"
 #include "event.h"
 
+struct env {
+	char	*env;
+	char	*value;
+};
+
 struct hyper_exec {
 	struct list_head	list;
-	struct hyper_event	e;
-	struct hyper_event	errev;
-	char			*id;
-	char			**argv;
-	int			argc;
-	uint64_t		seq;
-	uint64_t		errseq;
+	struct hyper_event	stdinev;
+	struct hyper_event	stdoutev;
+	struct hyper_event	stderrev;
 	int			pid;
 	int			ptyno;
 	int			init;
 	int			ptyfd;
-	int			errfd;
+	int			stdinfd;
+	int			stdoutfd;
+	int			stderrfd;
+	uint8_t			close_stdin_request;
 	uint8_t			code;
 	uint8_t			exit;
 	uint8_t			ref;
+
+	// configs
+	char			*id;
+	char			*user;
+	char			*group;
+	char			**additional_groups;
+	int			nr_additional_groups;
+	struct env		*envs;
+	int			envs_num;
+	char			**argv;
+	int			argc;
+	int			tty; // use tty or not
+	uint64_t		seq;
+	uint64_t		errseq;
+	char			*workdir;
 };
 
 struct hyper_pod;
 
 int hyper_exec_cmd(char *json, int length);
-int hyper_release_exec(struct hyper_exec *, struct hyper_pod *);
-int hyper_container_execcmd(struct hyper_pod *pod);
-int hyper_setup_exec_tty(struct hyper_exec *e);
-int hyper_dup_exec_tty(int fd, struct hyper_exec *e);
+int hyper_run_process(struct hyper_exec *e);
 struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid);
 struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq);
 int hyper_handle_exec_exit(struct hyper_pod *pod, int pid, uint8_t code);
-int hyper_watch_exec_pty(struct hyper_exec *exec, struct hyper_pod *pod);
 void hyper_cleanup_exec(struct hyper_pod *pod);
 
-extern struct hyper_event_ops pts_ops;
 #endif

src/hyper.h

@@ -2,12 +2,17 @@
 #define _HYPER_H_
 
 #include <stdint.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 #include "net.h"
 #include "list.h"
 #include "exec.h"
 #include "event.h"
 #include "container.h"
+#include "portmapping.h"
 
 enum {
 	RESERVED,
@@ -29,6 +34,10 @@ enum {
 	READFILE,
 	NEWCONTAINER,
 	KILLCONTAINER,
+	ONLINECPUMEM,
+	SETUPINTERFACE,
+	SETUPROUTE,
+	REMOVECONTAINER,
 };
 
 enum {
@@ -38,9 +47,9 @@ enum {
 };
 
 struct hyper_pod {
-	struct hyper_container	*c;
 	struct hyper_interface	*iface;
 	struct hyper_route	*rt;
+	struct portmapping_white_list	*portmap_white_lists;
 	char			**dns;
 	struct list_head	containers;
 	struct list_head	exec_head;
@@ -49,28 +58,27 @@ struct hyper_pod {
 	int			init_pid;
 	uint32_t		i_num;
 	uint32_t		r_num;
-	uint32_t		e_num;
 	uint32_t		d_num;
 	uint32_t		type;
 	/* how many containers are running */
 	uint32_t		remains;
 	uint8_t			policy;
 	int			efd;
-	struct hyper_event	sig;
+};
+
+struct portmapping_white_list {
+	char **internal_networks;
+	char **external_networks;
+	uint32_t i_num;
+	uint32_t e_num;
 };
 
 struct hyper_win_size {
-	char		*tty;
 	int		row;
 	int		column;
 	uint64_t	seq;
 };
 
-struct hyper_killer {
-	char		*id;
-	int		signal;
-};
-
 struct hyper_reader {
 	char		*id;
 	char		*file;
@@ -85,18 +93,35 @@ struct hyper_writter {
 
 struct hyper_ctl {
 	int			efd;
-	struct hyper_event	sig;
 	struct hyper_event	tty;
 	struct hyper_event	chan;
 };
 
-int hyper_mkdir(char *hyper_path);
+static inline int hyper_symlink(char *oldpath, char *newpath)
+{
+	return symlink(oldpath, newpath);
+}
+
+static inline int hyper_unlink(char *hyper_path)
+{
+	return unlink(hyper_path);
+}
+
+static inline int hyper_create(char *hyper_path)
+{
+	int fd = creat(hyper_path, 0755);
+	if (fd < 0)
+		return -1;
+
+	close(fd);
+	return 0;
+}
+
 int hyper_open_serial(char *tty);
-struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id);
-int hyper_start_containers(struct hyper_pod *pod);
 void hyper_cleanup_pod(struct hyper_pod *pod);
+int hyper_enter_sandbox(struct hyper_pod *pod, int pidpipe);
 
 extern struct hyper_pod global_pod;
 extern struct hyper_ctl ctl;
-extern struct hyper_exec *global_exec;
+extern sigset_t orig_mask;
 #endif

src/init.c

@@ -6,6 +6,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
+#include <sys/resource.h>
 #include <fcntl.h>
 #include <dirent.h>
 #include <sched.h>
@@ -15,7 +16,6 @@
 #include <limits.h>
 #include <mntent.h>
 #include <sys/epoll.h>
-#include <sys/signalfd.h>
 #include <inttypes.h>
 #include <ctype.h>
 
@@ -33,25 +33,22 @@ struct hyper_pod global_pod = {
 	.containers	=	LIST_HEAD_INIT(global_pod.containers),
 	.exec_head	=	LIST_HEAD_INIT(global_pod.exec_head),
 };
-struct hyper_exec *global_exec;
 
 #define MAXEVENTS	10
 
 struct hyper_ctl ctl;
 
-static struct hyper_event_ops hyper_signal_ops;
+sigset_t orig_mask;
+
 static int hyper_handle_exit(struct hyper_pod *pod);
 static int hyper_stop_pod(struct hyper_pod *pod);
 
 static int hyper_set_win_size(char *json, int length)
 {
-	struct hyper_win_size ws = {
-		.tty = NULL,
-	};
+	struct hyper_win_size ws;
 	struct winsize size;
 	struct hyper_exec *exec;
-	char path[128];
-	int fd, ret;
+	int ret;
 
 	fprintf(stdout, "call hyper_win_size, json %s, len %d\n", json, length);
 	if (hyper_parse_winsize(&ws, json, length) < 0) {
@@ -59,39 +56,19 @@ static int hyper_set_win_size(char *json, int length)
 		return -1;
 	}
 
-	if (!ws.tty) {
-		exec = hyper_find_exec_by_seq(&global_pod, ws.seq);
-		if (exec == NULL) {
-			fprintf(stdout, "can not find exec whose seq is %" PRIu64"\n", ws.seq);
-			return 0;
-		}
-
-		fprintf(stdout, "find exec %s, pid is %d, seq is %" PRIu64"\n",
-			exec->id ? exec->id : "pod", exec->pid, ws.seq);
-		fd = dup(exec->ptyfd);
-	} else {
-		if (sprintf(path, "/dev/%s", ws.tty) < 0) {
-			fprintf(stderr, "get tty device failed\n");
-			return -1;
-		}
-		fd = hyper_open_serial_dev(path);
-	}
-
-	if (fd < 0) {
-		perror("cannot open pty device to set term size");
-		goto out;
+	exec = hyper_find_exec_by_seq(&global_pod, ws.seq);
+	if (exec == NULL) {
+		fprintf(stdout, "can not find exec whose seq is %" PRIu64"\n", ws.seq);
+		return 0;
 	}
 
 	size.ws_row = ws.row;
 	size.ws_col = ws.column;
 
-	ret = ioctl(fd, TIOCSWINSZ, &size);
+	ret = ioctl(exec->ptyfd, TIOCSWINSZ, &size);
 	if (ret < 0)
 		perror("cannot ioctl to set pty device term size");
 
-	close(fd);
-out:
-	free(ws.tty);
 	return ret;
 }
 
@@ -145,7 +122,7 @@ static void hyper_term_all(struct hyper_pod *pod)
 	DIR *dp;
 	struct dirent *de;
 	pid_t *pids = NULL;
-	struct hyper_container *c;
+	struct hyper_exec *e;
 
 	dp = opendir("/proc");
 	if (dp == NULL)
@@ -177,8 +154,8 @@ static void hyper_term_all(struct hyper_pod *pod)
 	free(pids);
 	closedir(dp);
 
-	list_for_each_entry(c, &pod->containers, list)
-		hyper_kill_process(c->exec.pid);
+	list_for_each_entry(e, &pod->exec_head, list)
+		hyper_kill_process(e->pid);
 }
 
 static int hyper_handle_exit(struct hyper_pod *pod)
@@ -207,79 +184,14 @@ static int hyper_handle_exit(struct hyper_pod *pod)
 	return 0;
 }
 
-static int hyper_signal_loop(struct hyper_event *de)
+static void pod_init_sigchld(int sig)
 {
-	int size;
-	struct signalfd_siginfo sinfo;
-	struct hyper_pod *pod = de->ptr;
-
-	fprintf(stdout, "%s write to ctl tty fd\n", __func__);
-
-	while (1) {
-		size = read(de->fd, &sinfo, sizeof(struct signalfd_siginfo));
-		if (size <= 0) {
-			if (errno == EINTR)
-				continue;
-			if (errno == EAGAIN)
-				break;
-
-			perror("fail to read signal fd");
-			return -1;
-		} else if (size != sizeof(struct signalfd_siginfo)) {
-			perror("read signalfd siginfo failed");
-			return -1;
-		}
-
-		if (sinfo.ssi_signo != SIGCHLD) {
-			fprintf(stderr, "why give me signal %d?\n", sinfo.ssi_signo);
-			return 0;
-		}
-
-		hyper_handle_exit(pod);
-	}
-
-	return 0;
+	hyper_handle_exit(NULL);
 }
 
-static int pod_init_loop(struct hyper_pod *pod)
+static void hyper_init_sigchld(int sig)
 {
-	int i, n;
-	struct epoll_event *events;
-
-	pod->efd = epoll_create1(EPOLL_CLOEXEC);
-	if (pod->efd < 0) {
-		perror("epoll_create failed");
-		return -1;
-	}
-
-	fprintf(stdout, "hyper_init_event pod signal event %p, ops %p, fd %d\n",
-		&pod->sig, &hyper_signal_ops, pod->sig.fd);
-	if (hyper_init_event(&pod->sig, &hyper_signal_ops, NULL) < 0 ||
-	    hyper_add_event(pod->efd, &pod->sig, EPOLLIN) < 0) {
-		return -1;
-	}
-
-	events = calloc(MAXEVENTS, sizeof(*events));
-
-	while (1) {
-		n = epoll_wait(pod->efd, events, MAXEVENTS, -1);
-		fprintf(stdout, "%s epoll_wait %d\n", __func__, n);
-
-		if (n < 0) {
-			if (errno == EINTR)
-				continue;
-			perror("pod wait event failed");
-			return -1;
-		}
-
-		for (i = 0; i < n; i++) {
-			if (hyper_handle_event(pod->efd, &events[i]) < 0)
-				return -1;
-		}
-	}
-
-	close(pod->efd);
-	return 0;
+	hyper_handle_exit(&global_pod);
 }
 
 struct hyper_pod_arg {
@@ -294,26 +206,17 @@ static int hyper_pod_init(void *data)
 	sigset_t mask;
 
 	close(arg->ctl_pipe[0]);
-	close(ctl.sig.fd);
 	close(ctl.efd);
 	close(ctl.chan.fd);
 	close(ctl.tty.fd);
 
-	pod->sig.fd = -1;
-
 	sigemptyset(&mask);
 	sigaddset(&mask, SIGCHLD);
-
-	if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) {
+	if (sigprocmask(SIG_UNBLOCK, &mask, NULL) < 0) {
 		perror("sigprocmask SIGCHLD failed");
-		goto fail;
-	}
-
-	pod->sig.fd = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC);
-	if (pod->sig.fd < 0) {
-		perror("create pod signalfd failed");
-		goto fail;
+		return -1;
 	}
+	signal(SIGCHLD, pod_init_sigchld);
 
 	/* mount new proc directory */
 	if (umount("/proc") < 0) {
@@ -338,148 +241,36 @@ static int hyper_pod_init(void *data)
 
 	close(arg->ctl_pipe[1]);
 
-	pod_init_loop(pod);
-	fprintf(stdout, "pod init exit\n");
+	for (;;)
+		pause(); /* infinite loop and handle SIGCHLD */
 out:
 	_exit(-1);
 
 fail:
 	hyper_send_type(arg->ctl_pipe[1], ERROR);
 	close(arg->ctl_pipe[1]);
-	close(pod->sig.fd);
 
 	goto out;
 }
 
-struct hyper_stage0_arg {
-	struct hyper_pod	*pod;
-	struct hyper_container	*container;
-	int			ctl_pipe[2];
-};
-
-// stage0: enter the pidns
-static int hyper_container_stage0(void *data)
-{
-	int pidns, ipcns, utsns, ret;
-	struct hyper_container *c;
-	struct hyper_stage0_arg *arg;
-	struct hyper_pod *pod;
-	char path[64];
-
-	arg = data;
-	pod = arg->pod;
-	c   = arg->container;
-
-	ret = pidns = ipcns = utsns = -1;
-
-	sprintf(path, "/proc/%d/ns/pid", pod->init_pid);
-	pidns = open(path, O_RDONLY| O_CLOEXEC);
-	if (pidns < 0) {
-		perror("fail to open pidns of pod init");
-		goto out;
-	}
-
-	/* enter pidns of pod init, so the children of this process will run in
-	 * pidns of pod init, see man 2 setns */
-	if (setns(pidns, CLONE_NEWPID) < 0) {
-		perror("enter pidns of pod init failed");
-		goto out;
-	}
-
-	sprintf(path, "/proc/%d/ns/uts", pod->init_pid);
-	utsns = open(path, O_RDONLY| O_CLOEXEC);
-	if (utsns < 0) {
-		perror("fail to open utsns of pod init");
-		goto out;
-	}
-
-	sprintf(path, "/proc/%d/ns/ipc", pod->init_pid);
-	ipcns = open(path, O_RDONLY| O_CLOEXEC);
-	if (ipcns < 0) {
-		perror("fail to open ipcns of pod init");
-		goto out;
-	}
-
-	ret = hyper_start_container(c, utsns, ipcns, pod);
-out:
-	if (hyper_send_type(arg->ctl_pipe[1], ret ? ERROR : READY) < 0) {
-		fprintf(stderr, "container init send ready message failed\n");
-	}
-
-	close(pidns);
-	close(utsns);
-	close(ipcns);
-	/* hyper_container_stage0 shares fd table with init, let init closes pipe. */
-	//close(arg->ctl_pipe[0]);
-	//close(arg->ctl_pipe[1]);
-
-	_exit(ret);
-}
-
-int hyper_start_container_stage0(struct hyper_container *c, struct hyper_pod *pod)
-{
-	int stacksize = getpagesize() * 4;
-	void *stack = NULL;
-	struct hyper_stage0_arg arg = {
-		.pod		= pod,
-		.container	= c,
-		.ctl_pipe	= {-1, -1},
-	};
-	int ret = -1, pid;
-	uint32_t type;
-
-	if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) {
-		perror("create pipe between hyper init and pod init failed");
-		goto out;
-	}
-
-	stack = malloc(stacksize);
-	if (stack == NULL) {
-		perror("fail to allocate stack for container init");
-		goto out;
-	}
-
-	pid = clone(hyper_container_stage0, stack + stacksize, CLONE_VM| CLONE_FILES| SIGCHLD, &arg);
-	free(stack);
-	if (pid < 0) {
-		perror("enter container pid ns failed");
-		goto out;
-	}
-
-	/* Wait for container start */
-	if (hyper_get_type(arg.ctl_pipe[0], &type) < 0) {
-		perror("get enter_container_pidns ready message failed");
-		goto out;
-	}
-
-	if (type != READY) {
-		fprintf(stderr, "get incorrect enter_container_pidns message type %d, expect READY\n",
-			type);
-		goto out;
-	}
-
-	/* container process is spawned and ready to execute */
-	pod->remains++;
-	ret = 0;
-out:
-	close(arg.ctl_pipe[0]);
-	close(arg.ctl_pipe[1]);
-	return ret;
-}
-
-int hyper_start_containers(struct hyper_pod *pod)
+static int hyper_start_containers(struct hyper_pod *pod)
 {
 	struct hyper_container *c;
 
+	// TODO: setup containers and run container init processes
+	//       via separated hyperstart APIs
 	list_for_each_entry(c, &pod->containers, list) {
-		if (hyper_start_container_stage0(c, pod) < 0)
+		if (hyper_setup_container(c, pod) < 0)
 			return -1;
+		if (hyper_run_process(&c->exec) < 0)
+			return -1;
+		pod->remains++;
 	}
 
 	return 0;
 }
 
-static int hyper_setup_container(struct hyper_pod *pod)
+static int hyper_setup_pod_init(struct hyper_pod *pod)
 {
 	int stacksize = getpagesize() * 4;
 	int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC |
@@ -494,7 +285,7 @@ static int hyper_setup_container(struct hyper_pod *pod)
 	void *stack;
 	int ret = -1;
 
-	if (hyper_socketpair(PF_UNIX, SOCK_STREAM, 0, arg.ctl_pipe) < 0) {
+	if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) {
 		perror("create pipe between hyper init and pod init failed");
 		goto out;
 	}
@@ -526,11 +317,6 @@ static int hyper_setup_container(struct hyper_pod *pod)
 		goto out;
 	}
 
-	if (hyper_start_containers(pod) < 0) {
-		fprintf(stderr, "start containers failed\n");
-		goto out;
-	}
-
 	ret = 0;
 out:
 	close(arg.ctl_pipe[1]);
@@ -541,6 +327,63 @@ out:
 	return ret;
 }
 
+// enter the sanbox and pass to the child, shouldn't call from the init process
+int hyper_enter_sandbox(struct hyper_pod *pod, int pidpipe)
+{
+	int ret = -1, pidns = -1, utsns = -1, ipcns = -1;
+	char path[512];
+
+	sprintf(path, "/proc/%d/ns/pid", pod->init_pid);
+	pidns = open(path, O_RDONLY| O_CLOEXEC);
+	if (pidns < 0) {
+		perror("fail to open pidns of pod init");
+		goto out;
+	}
+
+	sprintf(path, "/proc/%d/ns/uts", pod->init_pid);
+	utsns = open(path, O_RDONLY| O_CLOEXEC);
+	if (utsns < 0) {
+		perror("fail to open utsns of pod init");
+		goto out;
+	}
+
+	sprintf(path, "/proc/%d/ns/ipc", pod->init_pid);
+	ipcns = open(path, O_RDONLY| O_CLOEXEC);
+	if (ipcns < 0) {
+		perror("fail to open ipcns of pod init");
+		goto out;
+	}
+
+	if (setns(pidns, CLONE_NEWPID) < 0 ||
+	    setns(utsns, CLONE_NEWUTS) < 0 ||
+	    setns(ipcns, CLONE_NEWIPC) < 0) {
+		perror("fail to enter the sandbox");
+		goto out;
+	}
+
+	/* current process isn't in the pidns even setns(pidns, CLONE_NEWPID)
+	 * was called. fork() is needed, so that the child process will run in
+	 * the pidns, see man 2 setns */
+	ret = fork();
+	if (ret < 0) {
+		perror("fail to fork");
+		goto out;
+	} else if (ret > 0) {
+		fprintf(stdout, "create child process pid=%d in the sandbox\n", ret);
+		if (pidpipe > 0) {
+			hyper_send_type(pidpipe, ret);
+		}
+		_exit(0);
+	}
+
+out:
+	close(pidns);
+	close(ipcns);
+	close(utsns);
+
+	return ret;
+}
+
 #ifdef WITH_VBOX
 
 #define MAX_HOST_NAME  256
@@ -577,7 +420,7 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 		return 0;
 	}
 
-	if (hyper_mkdir("/tmp/hyper/shared") < 0) {
+	if (hyper_mkdir("/tmp/hyper/shared", 0755) < 0) {
 		perror("fail to create /tmp/hyper/shared");
 		return -1;
 	}
@@ -608,13 +451,13 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 		return 0;
 	}
 
-	if (hyper_mkdir("/tmp/hyper/shared") < 0) {
+	if (hyper_mkdir("/tmp/hyper/shared", 0755) < 0) {
 		perror("fail to create /tmp/hyper/shared");
 		return -1;
 	}
 
 	if (mount(pod->share_tag, "/tmp/hyper/shared", "9p",
-		  MS_MGC_VAL| MS_NODEV, "trans=virtio,cache=loose") < 0) {
+		  MS_MGC_VAL| MS_NODEV, "trans=virtio") < 0) {
 
 		perror("fail to mount shared dir");
 		return -1;
@@ -627,7 +470,7 @@ static int hyper_setup_shared(struct hyper_pod *pod)
 static int hyper_setup_pod(struct hyper_pod *pod)
 {
 	/* create tmp proc directroy */
-	if (hyper_mkdir("/tmp/hyper/proc") < 0) {
+	if (hyper_mkdir("/tmp/hyper/proc", 0755) < 0) {
 		perror("create tmp proc failed");
 		return -1;
 	}
@@ -647,7 +490,12 @@ static int hyper_setup_pod(struct hyper_pod *pod)
 		return -1;
 	}
 
-	if (hyper_setup_container(pod) < 0) {
+	if (hyper_setup_portmapping(pod) < 0) {
+		fprintf(stderr, "setup port mapping failed\n");
+		return -1;
+	}
+
+	if (hyper_setup_pod_init(pod) < 0) {
 		fprintf(stderr, "start container failed\n");
 		return -1;
 	}
@@ -669,11 +517,11 @@ static void hyper_print_uptime(void)
 	close(fd);
 }
 
-static int hyper_destroy_pod(struct hyper_pod *pod)
+static int hyper_destroy_pod(struct hyper_pod *pod, int error)
 {
 	if (pod->init_pid == 0) {
 		/* Pod stopped, just shutdown */
-		hyper_shutdown();
+		hyper_shutdown(error);
 	} else {
 		/* Kill pod */
 		hyper_term_all(pod);
@@ -690,13 +538,20 @@ static int hyper_start_pod(char *json, int length)
 	if (pod->init_pid)
 		fprintf(stdout, "pod init_pid exist %d\n", pod->init_pid);
 
+	hyper_sync_time_hctosys();
 	if (hyper_parse_pod(pod, json, length) < 0) {
 		fprintf(stderr, "parse pod json failed\n");
 		return -1;
 	}
 
 	if (hyper_setup_pod(pod) < 0) {
-		hyper_destroy_pod(pod);
+		hyper_destroy_pod(pod, 1);
+		return -1;
+	}
+
+	if (hyper_start_containers(pod) < 0) {
+		fprintf(stderr, "start containers failed\n");
+		hyper_destroy_pod(pod, 1);
 		return -1;
 	}
 
@@ -723,35 +578,71 @@ static int hyper_new_container(char *json, int length)
 	}
 
 	list_add_tail(&c->list, &pod->containers);
-	ret = hyper_start_container_stage0(c, pod);
+	ret = hyper_setup_container(c, pod);
+	if (ret >= 0)
+		ret = hyper_run_process(&c->exec);
 	if (ret < 0) {
 		//TODO full grace cleanup
-		hyper_cleanup_container(c);
+		hyper_cleanup_container(c, pod);
 	}
+	pod->remains++;
 
 	return ret;
 }
 
 static int hyper_kill_container(char *json, int length)
 {
-	struct hyper_killer killer;
 	struct hyper_container *c;
 	struct hyper_pod *pod = &global_pod;
 	int ret = -1;
 
-	if (hyper_parse_kill_container(&killer, json, length) < 0) {
+	JSON_Value *value = hyper_json_parse(json, length);
+	if (value == NULL) {
 		goto out;
 	}
 
-	c = hyper_find_container(pod, killer.id);
+	const char *id = json_object_get_string(json_object(value), "container");
+	c = hyper_find_container(pod, id);
 	if (c == NULL) {
-		fprintf(stderr, "can not find container whose id is %s\n", killer.id);
+		fprintf(stderr, "can not find container whose id is %s\n", id);
 		goto out;
 	}
 
-	kill(c->exec.pid, killer.signal);
+	kill(c->exec.pid, (int)json_object_get_number(json_object(value), "signal"));
 	ret = 0;
 out:
+	json_value_free(value);
+	return ret;
+}
+
+static int hyper_remove_container(char *json, int length)
+{
+	struct hyper_container *c;
+	struct hyper_pod *pod = &global_pod;
+	int ret = -1;
+
+	JSON_Value *value = hyper_json_parse(json, length);
+	if (value == NULL) {
+		goto out;
+	}
+
+	const char *id = json_object_get_string(json_object(value), "container");
+	c = hyper_find_container(pod, id);
+	if (c == NULL) {
+		fprintf(stderr, "can not find container whose id is %s\n", id);
+		goto out;
+	}
+
+	if (c->exec.exit != 1) {
+		fprintf(stderr, "container %s has not been stopped\n", id);
+		goto out;
+	}
+
+	hyper_cleanup_container(c, pod);
+
+	ret = 0;
+out:
+	json_value_free(value);
 	return ret;
 }
 
@@ -762,7 +653,6 @@ static int hyper_cmd_write_file(char *json, int length)
 	struct hyper_pod *pod = &global_pod;
 	int pipe[2] = {-1, -1};
 	int pid, mntns = -1, fd;
-	char path[512];
 	int len = 0, size, ret = -1;
 
 	fprintf(stdout, "%s\n", __func__);
@@ -804,20 +694,14 @@ static int hyper_cmd_write_file(char *json, int length)
 		goto out;
 	}
 
+	/* TODO: wait for container finishing setup root */
 	if (setns(mntns, CLONE_NEWNS) < 0) {
 		perror("fail to enter container ns");
 		goto exit;
 	}
 
-	sprintf(path, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs);
 	fprintf(stdout, "write file %s, data len %d\n", writter.file, writter.len);
 
-	/* TODO: wait for container finishing setup root */
-	if (chroot(path) < 0) {
-		perror("chroot for exec command failed");
-		goto exit;
-	}
-
 	fd = open(writter.file, O_CREAT| O_TRUNC| O_WRONLY, 0644);
 	if (fd < 0) {
 		perror("fail to open target file");
@@ -866,6 +750,7 @@ static int hyper_do_cmd_read_file(void *data)
 	int len = 0, size, fd, ret = -1;
 	struct hyper_file_arg *arg = data;
 
+	/* TODO: wait for container finishing setup root */
 	if (setns(arg->mntns, CLONE_NEWNS) < 0) {
 		perror("fail to enter container ns");
 		goto err;
@@ -873,12 +758,6 @@ static int hyper_do_cmd_read_file(void *data)
 
 	fprintf(stdout, "read file %s\n", arg->file);
 
-	/* TODO: wait for container finishing setup root */
-	if (chroot(arg->root) < 0) {
-		perror("chroot for exec command failed");
-		goto err;
-	}
-
 	if (stat(arg->file, &st) < 0) {
 		perror("fail to state file");
 		goto err;
@@ -916,7 +795,7 @@ static int hyper_do_cmd_read_file(void *data)
 	ret = 0;
 err:
 	hyper_send_type(arg->pipe[1], ret ? ERROR : READY);
-	return ret;
+	_exit(ret);
 }
 
 static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_t **data)
@@ -929,7 +808,7 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_
 	};
 	int stacksize = getpagesize() * 4;
 	void *stack = NULL;
-	int pid, ret = -1;
+	int pid, ret = -1, status;
 	uint32_t type;
 
 	fprintf(stdout, "%s\n", __func__);
@@ -958,7 +837,6 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_
 	arg.file = reader.file;
 	arg.datalen = datalen;
 	arg.data = data;
-	sprintf(arg.root, "/tmp/hyper/%s/root/%s/", c->id, c->rootfs);
 
 	stack = malloc(stacksize);
 	if (stack == NULL) {
@@ -966,13 +844,17 @@ static int hyper_cmd_read_file(char *json, int length, uint32_t *datalen, uint8_
 		goto out;
 	}
 
-	pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM| SIGCHLD, &arg);
-	free(stack);
+	pid = clone(hyper_do_cmd_read_file, stack + stacksize, CLONE_VM| SIGQUIT, &arg);
 	if (pid < 0) {
 		perror("fail to fork writter process");
 		goto out;
 	}
 
+	if (waitpid(pid, &status, __WCLONE) <= 0) {
+		perror("waiting hyper_do_cmd_read_file finish failed");
+		goto out;
+	}
+
 	if (hyper_get_type(arg.pipe[0], &type) < 0 || type != READY) {
 		fprintf(stderr, "%s to incorrect type %" PRIu32 "\n", __func__, type);
 		goto out;
@@ -984,10 +866,23 @@ out:
 	close(arg.pipe[1]);
 	free(reader.id);
 	free(reader.file);
+	free(stack);
 
 	return ret;
 }
 
+static void hyper_cmd_online_cpu_mem()
+{
+	int pid = fork();
+	if (pid < 0) {
+		perror("fail to fork online process");
+	} else if (pid == 0) {
+		online_cpu();
+		online_memory();
+		exit(0);
+	}
+}
+
 static void hyper_cleanup_hostname(struct hyper_pod *pod)
 {
 	free(pod->hostname);
@@ -1025,6 +920,7 @@ void hyper_cleanup_pod(struct hyper_pod *pod)
 	hyper_cleanup_network(pod);
 	hyper_cleanup_shared(pod);
 	hyper_cleanup_dns(pod);
+	hyper_cleanup_portmapping(pod);
 	hyper_cleanup_hostname(pod);
 }
 
@@ -1107,12 +1003,12 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len)
 	dprintf(stdout, "find exec %s pid %d, seq is %" PRIu64 "\n",
 		exec->id ? exec->id : "pod", exec->pid, exec->seq);
 	// if exec is exited, the event fd of exec is invalid. don't accept any input.
-	if (exec->exit) {
+	if (exec->exit || exec->close_stdin_request) {
 		fprintf(stdout, "exec seq %" PRIu64 " exited, don't accept any input\n", exec->seq);
 		return 0;
 	}
 
-	wbuf = &exec->e.wbuf;
+	wbuf = &exec->stdinev.wbuf;
 
 	size = wbuf->size - wbuf->get;
 	if (size == 0)
@@ -1123,10 +1019,20 @@ static int hyper_ttyfd_handle(struct hyper_event *de, uint32_t len)
 	if (size > (len - 12))
 		size = (len - 12);
 
+	/* size == 0 means we had received eof */
+	if (size == 0 && !exec->tty) {
+		exec->close_stdin_request = 1;
+		/* we can't hup the stdinev here, force hup on next write */
+		if (hyper_modify_event(ctl.efd, &exec->stdinev, EPOLLOUT) < 0) {
+			fprintf(stderr, "modify exec pts event to in & out failed\n");
+			return -1;
+		}
+	}
+
 	if (size > 0) {
 		memcpy(wbuf->data + wbuf->get, rbuf->data + 12, size);
 		wbuf->get += size;
-		if (hyper_modify_event(ctl.efd, &exec->e, EPOLLIN | EPOLLOUT) < 0) {
+		if (hyper_modify_event(ctl.efd, &exec->stdinev, EPOLLOUT) < 0) {
 			fprintf(stderr, "modify exec pts event to in & out failed\n");
 			return -1;
 		}
@@ -1163,7 +1069,7 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 		//break;
 	case DESTROYPOD:
 		fprintf(stdout, "get DESTROYPOD message\n");
-		hyper_destroy_pod(pod);
+		hyper_destroy_pod(pod, 0);
 		return 0;
 	case EXECCMD:
 		ret = hyper_exec_cmd((char *)buf->data + 8, len - 8);
@@ -1189,6 +1095,18 @@ static int hyper_channel_handle(struct hyper_event *de, uint32_t len)
 	case KILLCONTAINER:
 		ret = hyper_kill_container((char *)buf->data + 8, len - 8);
 		break;
+	case REMOVECONTAINER:
+		ret = hyper_remove_container((char *)buf->data + 8, len - 8);
+		break;
+	case ONLINECPUMEM:
+		hyper_cmd_online_cpu_mem();
+		break;
+	case SETUPINTERFACE:
+		ret = hyper_cmd_setup_interface((char *)buf->data + 8, len - 8);
+		break;
+	case SETUPROUTE:
+		ret = hyper_cmd_setup_route((char *)buf->data + 8, len - 8);
+		break;
 	default:
 		ret = -1;
 		break;
@@ -1221,16 +1139,56 @@ static struct hyper_event_ops hyper_ttyfd_ops = {
 	.len_offset	= 8,
 };
 
-static struct hyper_event_ops hyper_signal_ops = {
-	.read		= hyper_signal_loop,
-	.hup		= hyper_event_hup,
-};
-
 static int hyper_loop(void)
 {
 	int i, n;
 	struct epoll_event *events;
 	struct hyper_pod *pod = &global_pod;
+	sigset_t mask, omask;
+	struct rlimit limit;
+	char *filemax = "1000000";
+
+	sigemptyset(&mask);
+	sigaddset(&mask, SIGCHLD);
+
+	/*
+	 * block SIGCHLD in the loop except when in the syscall of
+	 * epoll_pwait(), it ensures that the SIGCHLD handling and
+	 * the events handling are exclusive.
+	 */
+	if (sigprocmask(SIG_BLOCK, &mask, &omask) < 0) {
+		perror("sigprocmask SIGCHLD failed");
+		return -1;
+	}
+	// need original mask to restore sigmask of child processes
+	orig_mask = omask;
+	sigdelset(&omask, SIGCHLD);
+	signal(SIGCHLD, hyper_init_sigchld);
+
+	if (hyper_write_file("/proc/sys/fs/file-max", filemax, strlen(filemax)) < 0) {
+		fprintf(stderr, "sysctl: setup default file-max(%s) failed\n", filemax);
+		return -1;
+	}
+
+	// setup open file limit
+	limit.rlim_cur = limit.rlim_max = atoi(filemax);
+	if (setrlimit(RLIMIT_NOFILE, &limit) < 0) {
+		perror("set rlimit for NOFILE failed");
+		return -1;
+	}
+
+	// setup process num limit
+	limit.rlim_cur = limit.rlim_max = 30604;
+	if (setrlimit(RLIMIT_NPROC, &limit) < 0) {
+		perror("set rlimit for NPROC failed");
+		return -1;
+	}
+
+	// setup pending signal limit, same with NRPROC
+	if (setrlimit(RLIMIT_SIGPENDING, &limit) < 0) {
+		perror("set rlimit for SIGPENDING failed");
+		return -1;
+	}
 
 	ctl.efd = epoll_create1(EPOLL_CLOEXEC);
 	if (ctl.efd < 0) {
@@ -1252,17 +1210,10 @@ static int hyper_loop(void)
 		return -1;
 	}
 
-	fprintf(stdout, "hyper_init_event hyper signal event %p, ops %p, fd %d\n",
-		&ctl.sig, &hyper_signal_ops, ctl.sig.fd);
-	if (hyper_init_event(&ctl.sig, &hyper_signal_ops, pod) < 0 ||
-	    hyper_add_event(ctl.efd, &ctl.sig, EPOLLIN) < 0) {
-		return -1;
-	}
-
 	events = calloc(MAXEVENTS, sizeof(*events));
 
 	while (1) {
-		n = epoll_wait(ctl.efd, events, MAXEVENTS, -1);
+		n = epoll_pwait(ctl.efd, events, MAXEVENTS, -1, &omask);
 		fprintf(stdout, "%s epoll_wait %d\n", __func__, n);
 
 		if (n < 0) {
@@ -1285,11 +1236,11 @@ static int hyper_loop(void)
 int main(int argc, char *argv[])
 {
 	char *cmdline, *ctl_serial, *tty_serial;
-	sigset_t mask;
 
-	if (hyper_mkdir("/dev") < 0 ||
-	    hyper_mkdir("/sys") < 0 ||
-	    hyper_mkdir("/proc") < 0) {
+	if (hyper_mkdir("/dev", 0755) < 0 ||
+	    hyper_mkdir("/sys", 0755) < 0 ||
+	    hyper_mkdir("/sbin", 0755) < 0 ||
+	    hyper_mkdir("/proc", 0755) < 0) {
 		perror("create basic directroy failed");
 		return -1;
 	}
@@ -1311,7 +1262,7 @@ int main(int argc, char *argv[])
 		return -1;
 	}
 
-	if (hyper_mkdir("/dev/pts") < 0) {
+	if (hyper_mkdir("/dev/pts", 0755) < 0) {
 		perror("create basic directroy failed");
 		return -1;
 	}
@@ -1321,6 +1272,14 @@ int main(int argc, char *argv[])
 		return -1;
 	}
 
+	symlink("/busybox", "/sh");
+	symlink("/busybox", "/tar");
+	symlink("/busybox", "/sbin/modprobe");
+	symlink("/busybox", "/sbin/depmod");
+	symlink("/iptables", "/sbin/iptables");
+	symlink("/iptables", "/sbin/iptables-restore");
+	symlink("/iptables", "/sbin/iptables-save");
+
 	cmdline = read_cmdline();
 
 	setsid();
@@ -1343,20 +1302,6 @@ int main(int argc, char *argv[])
 
 	setenv("PATH", "/bin:/sbin/:/usr/bin/:/usr/sbin/", 1);
 
-	sigemptyset(&mask);
-	sigaddset(&mask, SIGCHLD);
-
-	if (sigprocmask(SIG_BLOCK, &mask, NULL) < 0) {
-		perror("sigprocmask SIGCHLD failed");
-		return -1;
-	}
-
-	ctl.sig.fd = signalfd(-1, &mask, SFD_NONBLOCK | SFD_CLOEXEC);
-	if (ctl.sig.fd < 0) {
-		perror("create signalfd failed");
-		return -1;
-	}
-
 	ctl.chan.fd = hyper_setup_ctl_channel(ctl_serial);
 	if (ctl.chan.fd < 0) {
 		fprintf(stderr, "fail to setup hyper control serial port\n");
@@ -1375,8 +1320,6 @@ int main(int argc, char *argv[])
 out2:
 	close(ctl.chan.fd);
 out1:
-	close(ctl.sig.fd);
-
 	free(cmdline);
 
 	return 0;

src/net.c

@@ -11,6 +11,7 @@
 
 #include "hyper.h"
 #include "util.h"
+#include "parse.h"
 #include "../config.h"
 
 void hyper_set_be32(uint8_t *buf, uint32_t val)
@@ -783,6 +784,69 @@ void hyper_cleanup_network(struct hyper_pod *pod)
 	netlink_close(&rth);
 }
 
+int hyper_cmd_setup_interface(char *json, int length)
+{
+	int ret = -1;
+	struct hyper_interface *iface;
+	struct rtnl_handle rth;
+
+	if (hyper_rescan() < 0)
+		return -1;
+
+	if (netlink_open(&rth) < 0)
+		return -1;
+
+
+	iface = hyper_parse_setup_interface(json, length);
+	if (iface == NULL) {
+		fprintf(stderr, "parse interface failed\n");
+		goto out;
+	}
+	ret = hyper_setup_interface(&rth, iface);
+	if (ret < 0) {
+		fprintf(stderr, "link up device %s failed\n", iface->device);
+		goto out1;
+	}
+	ret = 0;
+out1:
+	free(iface->device);
+	free(iface->ipaddr);
+	free(iface->mask);
+	free(iface);
+out:
+	netlink_close(&rth);
+	return ret;
+}
+
+int hyper_cmd_setup_route(char *json, int length) {
+	struct hyper_route *rts = NULL;
+	int i, ret = -1;
+	uint32_t r_num;
+	struct rtnl_handle rth;
+
+	if (netlink_open(&rth) < 0)
+		return -1;
+
+	if (hyper_parse_setup_routes(&rts, &r_num, json, length) < 0) {
+		fprintf(stderr, "parse route failed\n");
+		goto out;
+	}
+
+	for (i = 0; i < r_num; i++) {
+		ret = hyper_setup_route(&rth, &rts[i]);
+		if (ret < 0) {
+			fprintf(stderr, "setup route failed\n");
+			goto out;
+		}
+	}
+
+	ret = 0;
+out:
+	netlink_close(&rth);
+	free(rts);
+	return ret;
+}
+
 int hyper_setup_dns(struct hyper_pod *pod)
 {
 	int i, fd, ret = -1;

src/net.h

@@ -47,6 +47,8 @@ uint32_t hyper_get_be32(uint8_t *buf);
 void hyper_set_be64(uint8_t *buf, uint64_t val);
 uint64_t hyper_get_be64(uint8_t *buf);
 int hyper_setup_network(struct hyper_pod *pod);
+int hyper_cmd_setup_interface(char *json, int length);
+int hyper_cmd_setup_route(char *json, int length);
 void hyper_cleanup_network(struct hyper_pod *pod);
 int hyper_setup_dns(struct hyper_pod *pod);
 void hyper_cleanup_dns(struct hyper_pod *pod);

src/parse.h

@@ -3,16 +3,19 @@
 
 #include "hyper.h"
 #include "jsmn.h"
+#include "parson.h"
 
 int hyper_parse_pod(struct hyper_pod *pod, char *json, int length);
 struct hyper_exec *hyper_parse_execcmd(char *json, int length);
 char *json_token_str(char *js, jsmntok_t *t);
 int json_token_streq(char *js, jsmntok_t *t, char *s);
 int hyper_parse_winsize(struct hyper_win_size *ws, char *json, int length);
-int hyper_parse_kill_container(struct hyper_killer *killer, char *json, int length);
 int hyper_parse_write_file(struct hyper_writter *writter, char *json, int length);
 int hyper_parse_read_file(struct hyper_reader *reader, char *json, int length);
 struct hyper_container *hyper_parse_new_container(struct hyper_pod *pod, char *json, int length);
 void hyper_free_container(struct hyper_container *c);
+struct hyper_interface *hyper_parse_setup_interface(char *json, int length);
+int hyper_parse_setup_routes(struct hyper_route **routes, uint32_t *r_num, char *json, int length);
+JSON_Value *hyper_json_parse(char *json, int length);
 
 #endif

src/parson.h

@@ -0,0 +1,223 @@
+/*
+ Parson ( http://kgabis.github.com/parson/ )
+ Copyright (c) 2012 - 2016 Krzysztof Gabis
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+*/
+
+#ifndef parson_parson_h
+#define parson_parson_h
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include <stddef.h>   /* size_t */
+
+/* Types and enums */
+typedef struct json_object_t JSON_Object;
+typedef struct json_array_t  JSON_Array;
+typedef struct json_value_t  JSON_Value;
+
+enum json_value_type {
+    JSONError   = -1,
+    JSONNull    = 1,
+    JSONString  = 2,
+    JSONNumber  = 3,
+    JSONObject  = 4,
+    JSONArray   = 5,
+    JSONBoolean = 6
+};
+typedef int JSON_Value_Type;
+
+enum json_result_t {
+    JSONSuccess = 0,
+    JSONFailure = -1
+};
+typedef int JSON_Status;
+
+typedef void * (*JSON_Malloc_Function)(size_t);
+typedef void   (*JSON_Free_Function)(void *);
+
+/* Call only once, before calling any other function from parson API. If not called, malloc and free
+   from stdlib will be used for all allocations */
+void json_set_allocation_functions(JSON_Malloc_Function malloc_fun, JSON_Free_Function free_fun);
+
+/* Parses first JSON value in a file, returns NULL in case of error */
+JSON_Value * json_parse_file(const char *filename);
+
+/* Parses first JSON value in a file and ignores comments (/ * * / and //),
+   returns NULL in case of error */
+JSON_Value * json_parse_file_with_comments(const char *filename);
+
+/*  Parses first JSON value in a string, returns NULL in case of error */
+JSON_Value * json_parse_string(const char *string);
+
+/*  Parses first JSON value in a string and ignores comments (/ * * / and //),
+    returns NULL in case of error */
+JSON_Value * json_parse_string_with_comments(const char *string);
+
+/* Serialization */
+size_t      json_serialization_size(const JSON_Value *value); /* returns 0 on fail */
+JSON_Status json_serialize_to_buffer(const JSON_Value *value, char *buf, size_t buf_size_in_bytes);
+JSON_Status json_serialize_to_file(const JSON_Value *value, const char *filename);
+char *      json_serialize_to_string(const JSON_Value *value);
+
+/* Pretty serialization */
+size_t      json_serialization_size_pretty(const JSON_Value *value); /* returns 0 on fail */
+JSON_Status json_serialize_to_buffer_pretty(const JSON_Value *value, char *buf, size_t buf_size_in_bytes);
+JSON_Status json_serialize_to_file_pretty(const JSON_Value *value, const char *filename);
+char *      json_serialize_to_string_pretty(const JSON_Value *value);
+
+void        json_free_serialized_string(char *string); /* frees string from json_serialize_to_string and json_serialize_to_string_pretty */
+
+/* Comparing */
+int  json_value_equals(const JSON_Value *a, const JSON_Value *b);
+
+/* Validation
+   This is *NOT* JSON Schema. It validates json by checking if object have identically
+   named fields with matching types.
+   For example schema {"name":"", "age":0} will validate
+   {"name":"Joe", "age":25} and {"name":"Joe", "age":25, "gender":"m"},
+   but not {"name":"Joe"} or {"name":"Joe", "age":"Cucumber"}.
+   In case of arrays, only first value in schema is checked against all values in tested array.
+   Empty objects ({}) validate all objects, empty arrays ([]) validate all arrays,
+   null validates values of every type.
+ */
+JSON_Status json_validate(const JSON_Value *schema, const JSON_Value *value);
+
+/*
+ * JSON Object
+ */
+JSON_Value  * json_object_get_value  (const JSON_Object *object, const char *name);
+const char  * json_object_get_string (const JSON_Object *object, const char *name);
+JSON_Object * json_object_get_object (const JSON_Object *object, const char *name);
+JSON_Array  * json_object_get_array  (const JSON_Object *object, const char *name);
+double        json_object_get_number (const JSON_Object *object, const char *name); /* returns 0 on fail */
+int           json_object_get_boolean(const JSON_Object *object, const char *name); /* returns -1 on fail */
+
+/* dotget functions enable addressing values with dot notation in nested objects,
+ just like in structs or c++/java/c# objects (e.g. objectA.objectB.value).
+ Because valid names in JSON can contain dots, some values may be inaccessible
+ this way. */
+JSON_Value  * json_object_dotget_value  (const JSON_Object *object, const char *name);
+const char  * json_object_dotget_string (const JSON_Object *object, const char *name);
+JSON_Object * json_object_dotget_object (const JSON_Object *object, const char *name);
+JSON_Array  * json_object_dotget_array  (const JSON_Object *object, const char *name);
+double        json_object_dotget_number (const JSON_Object *object, const char *name); /* returns 0 on fail */
+int           json_object_dotget_boolean(const JSON_Object *object, const char *name); /* returns -1 on fail */
+
+/* Functions to get available names */
+size_t        json_object_get_count   (const JSON_Object *object);
+const char  * json_object_get_name    (const JSON_Object *object, size_t index);
+JSON_Value  * json_object_get_value_at(const JSON_Object *object, size_t index);
+
+/* Creates new name-value pair or frees and replaces old value with a new one.
+ * json_object_set_value does not copy passed value so it shouldn't be freed afterwards. */
+JSON_Status json_object_set_value(JSON_Object *object, const char *name, JSON_Value *value);
+JSON_Status json_object_set_string(JSON_Object *object, const char *name, const char *string);
+JSON_Status json_object_set_number(JSON_Object *object, const char *name, double number);
+JSON_Status json_object_set_boolean(JSON_Object *object, const char *name, int boolean);
+JSON_Status json_object_set_null(JSON_Object *object, const char *name);
+
+/* Works like dotget functions, but creates whole hierarchy if necessary.
+ * json_object_dotset_value does not copy passed value so it shouldn't be freed afterwards. */
+JSON_Status json_object_dotset_value(JSON_Object *object, const char *name, JSON_Value *value);
+JSON_Status json_object_dotset_string(JSON_Object *object, const char *name, const char *string);
+JSON_Status json_object_dotset_number(JSON_Object *object, const char *name, double number);
+JSON_Status json_object_dotset_boolean(JSON_Object *object, const char *name, int boolean);
+JSON_Status json_object_dotset_null(JSON_Object *object, const char *name);
+
+/* Frees and removes name-value pair */
+JSON_Status json_object_remove(JSON_Object *object, const char *name);
+
+/* Works like dotget function, but removes name-value pair only on exact match. */
+JSON_Status json_object_dotremove(JSON_Object *object, const char *key);
+
+/* Removes all name-value pairs in object */
+JSON_Status json_object_clear(JSON_Object *object);
+
+/*
+ *JSON Array
+ */
+JSON_Value  * json_array_get_value  (const JSON_Array *array, size_t index);
+const char  * json_array_get_string (const JSON_Array *array, size_t index);
+JSON_Object * json_array_get_object (const JSON_Array *array, size_t index);
+JSON_Array  * json_array_get_array  (const JSON_Array *array, size_t index);
+double        json_array_get_number (const JSON_Array *array, size_t index); /* returns 0 on fail */
+int           json_array_get_boolean(const JSON_Array *array, size_t index); /* returns -1 on fail */
+size_t        json_array_get_count  (const JSON_Array *array);
+
+/* Frees and removes value at given index, does nothing and returns JSONFailure if index doesn't exist.
+ * Order of values in array may change during execution.  */
+JSON_Status json_array_remove(JSON_Array *array, size_t i);
+
+/* Frees and removes from array value at given index and replaces it with given one.
+ * Does nothing and returns JSONFailure if index doesn't exist.
+ * json_array_replace_value does not copy passed value so it shouldn't be freed afterwards. */
+JSON_Status json_array_replace_value(JSON_Array *array, size_t i, JSON_Value *value);
+JSON_Status json_array_replace_string(JSON_Array *array, size_t i, const char* string);
+JSON_Status json_array_replace_number(JSON_Array *array, size_t i, double number);
+JSON_Status json_array_replace_boolean(JSON_Array *array, size_t i, int boolean);
+JSON_Status json_array_replace_null(JSON_Array *array, size_t i);
+
+/* Frees and removes all values from array */
+JSON_Status json_array_clear(JSON_Array *array);
+
+/* Appends new value at the end of array.
+ * json_array_append_value does not copy passed value so it shouldn't be freed afterwards. */
+JSON_Status json_array_append_value(JSON_Array *array, JSON_Value *value);
+JSON_Status json_array_append_string(JSON_Array *array, const char *string);
+JSON_Status json_array_append_number(JSON_Array *array, double number);
+JSON_Status json_array_append_boolean(JSON_Array *array, int boolean);
+JSON_Status json_array_append_null(JSON_Array *array);
+
+/*
+ *JSON Value
+ */
+JSON_Value * json_value_init_object (void);
+JSON_Value * json_value_init_array  (void);
+JSON_Value * json_value_init_string (const char *string); /* copies passed string */
+JSON_Value * json_value_init_number (double number);
+JSON_Value * json_value_init_boolean(int boolean);
+JSON_Value * json_value_init_null   (void);
+JSON_Value * json_value_deep_copy   (const JSON_Value *value);
+void         json_value_free        (JSON_Value *value);
+
+JSON_Value_Type json_value_get_type   (const JSON_Value *value);
+JSON_Object *   json_value_get_object (const JSON_Value *value);
+JSON_Array  *   json_value_get_array  (const JSON_Value *value);
+const char  *   json_value_get_string (const JSON_Value *value);
+double          json_value_get_number (const JSON_Value *value);
+int             json_value_get_boolean(const JSON_Value *value);
+
+/* Same as above, but shorter */
+JSON_Value_Type json_type   (const JSON_Value *value);
+JSON_Object *   json_object (const JSON_Value *value);
+JSON_Array  *   json_array  (const JSON_Value *value);
+const char  *   json_string (const JSON_Value *value);
+double          json_number (const JSON_Value *value);
+int             json_boolean(const JSON_Value *value);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/portmapping.c

@@ -0,0 +1,410 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <limits.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/wait.h>
+#include <sys/utsname.h>
+#include <arpa/inet.h>
+#include <fcntl.h>
+
+#include "hyper.h"
+#include "util.h"
+#include "../config.h"
+
+int hyper_init_modules() 
+{
+	int status = hyper_cmd("/sbin/depmod");
+	if (status != 0) {
+		fprintf(stderr, "depmod failed, status: %d\n", status);
+		return -1;
+	}
+
+	return 0;
+}
+
+int hyper_setup_iptables_rule(struct ipt_rule rule)
+{
+	char check_cmd[512] = {0};
+	char cmd[512] = {0};
+	int check = -1;
+
+	if (rule.rule != NULL) {
+		sprintf(check_cmd, "/sbin/iptables -t %s -C %s %s", rule.table, rule.chain, rule.rule);
+		sprintf(cmd, "/sbin/iptables -t %s %s %s %s", rule.table, rule.op, rule.chain, rule.rule);
+	} else {
+		sprintf(cmd, "/sbin/iptables -t %s %s %s", rule.table, rule.op, rule.chain);
+	}
+
+	if (strlen(check_cmd) > 0) {
+		check = hyper_cmd(check_cmd);
+		fprintf(stdout, "check iptables '%s', ret: %d\n", check_cmd, check);
+	}
+
+	if (check == 0) {
+		// iptables rule already exist, do not insert it again
+		if (!strncmp(rule.op, "-A", strlen("-A")) ||
+			!strncmp(rule.op, "-I", strlen("-I")) ||
+			!strncmp(rule.op, "-N", strlen("-N"))) {
+			fprintf(stdout, "iptables rule '%s' already exist\n", rule.rule);
+			return 0;
+		}
+	}
+
+	int status = hyper_cmd(cmd);
+	fprintf(stdout, "insert iptables '%s', ret: %d\n", cmd, status);
+	if (status != 0) {
+		fprintf(stderr, "insert iptables rule failed, ret: %d\n", status);
+		return -1;
+	}
+
+	return 0;
+}
+
+// initialize modules and iptables chains
+int hyper_setup_portmapping(struct hyper_pod *pod)
+{
+	if (pod->portmap_white_lists == NULL || (pod->portmap_white_lists->i_num == 0 &&
+			pod->portmap_white_lists->e_num == 0)) {
+		return 0;
+	}
+
+	if (hyper_init_modules() < 0) {
+		return -1;
+	}
+
+	// iptables -t filter -N hyperstart-INPUT
+	// iptables -t nat -N hyperstart-PREROUTING
+	// iptables -t filter -I INPUT -j hyperstart-INPUT
+	// iptables -t nat -I PREROUTING -j hyperstart-PREROUTING
+	// iptables -t filter -A hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+	// iptables -t filter -A hyperstart-INPUT -p icmp -j ACCEPT
+	// iptables -t filter -A hyperstart-INPUT -i lo -j ACCEPT
+	// iptables -t filter -A hyperstart-INPUT -j DROP
+	// iptables -t nat -A hyperstart-PREROUTING -j RETURN
+	const struct ipt_rule rules[] = {
+		{
+			.table = "filter",
+			.op = "-N",
+			.chain = "hyperstart-INPUT",
+			.rule = NULL,
+		},
+		{
+			.table = "nat",
+			.op = "-N",
+			.chain = "hyperstart-PREROUTING",
+			.rule = NULL,
+		},
+		{
+			.table = "filter",
+			.op = "-I",
+			.chain = "INPUT",
+			.rule = "-j hyperstart-INPUT",
+		},
+		{
+			.table = "nat",
+			.op = "-I",
+			.chain = "PREROUTING",
+			.rule = "-j hyperstart-PREROUTING",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-m state --state RELATED,ESTABLISHED -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-p icmp -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-i lo -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-j DROP",
+		},
+		{
+			.table = "nat",
+			.op = "-A",
+			.chain = "hyperstart-PREROUTING",
+			.rule = "-j RETURN",
+		},
+	};
+
+	int i = 0;
+	for(i=0; i< sizeof(rules)/sizeof(struct ipt_rule); i++) {
+		if (hyper_setup_iptables_rule(rules[i])<0) {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+void hyper_cleanup_portmapping(struct hyper_pod *pod)
+{
+	if (pod->portmap_white_lists == NULL || (pod->portmap_white_lists->i_num == 0 &&
+			pod->portmap_white_lists->e_num == 0)) {
+		return;
+	}
+
+	// iptables -t filter -D hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+	// iptables -t filter -D hyperstart-INPUT -p icmp -j ACCEPT
+	// iptables -t filter -D hyperstart-INPUT -i lo -j ACCEPT
+	// iptables -t filter -D hyperstart-INPUT -j DROP
+	// iptables -t filter -D INPUT -j hyperstart-DNPUT
+	// iptables -t nat -D hyperstart-PREROUTING -j RETURN
+	// iptables -t nat -D PREROUTING -j hyperstart-PREROUTING
+	// iptables -t filter -F hyperstart-INPUT
+	// iptables -t nat -F hyperstart-PREROUTING
+	// iptables -t filter -X hyperstart-INPUT
+	// iptables -t nat -X hyperstart-PREROUTING
+	const struct ipt_rule rules[] = {
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-m state --state RELATED,ESTABLISHED -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-p icmp -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-i lo -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-j DROP",
+		},
+		{
+			.table = "nat",
+			.op = "-D",
+			.chain = "hyperstart-PREROUTING",
+			.rule = "-j RETURN",
+		},
+		{
+			.table = "nat",
+			.op = "-D",
+			.chain = "PREROUTING",
+			.rule = "-j hyperstart-PREROUTING",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "INPUT",
+			.rule = "-j hyperstart-INPUT",
+		},
+		{
+			.table = "nat",
+			.op = "-F",
+			.chain = "hyperstart-PREROUTING",
+			.rule = NULL,
+		},
+		{
+			.table = "nat",
+			.op = "-X",
+			.chain = "hyperstart-PREROUTING",
+			.rule = NULL,
+		},
+		{
+			.table = "filter",
+			.op = "-F",
+			.chain = "hyperstart-INPUT",
+			.rule = NULL,
+		},
+		{
+			.table = "filter",
+			.op = "-X",
+			.chain = "hyperstart-INPUT",
+			.rule = NULL,
+		},
+	};
+
+	int i = 0;
+	for(i=0; i< sizeof(rules)/sizeof(struct ipt_rule); i++) {
+		if (hyper_setup_iptables_rule(rules[i])<0) {
+			return;
+		}
+	}
+
+	free(pod->portmap_white_lists->internal_networks);
+	free(pod->portmap_white_lists->external_networks);
+	free(pod->portmap_white_lists);
+	pod->portmap_white_lists = NULL;
+}
+
+int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod)
+{
+	if (pod->portmap_white_lists == NULL || (pod->portmap_white_lists->i_num == 0 &&
+			pod->portmap_white_lists->e_num == 0)) {
+		return 0;
+	}
+
+	if (c->ports_num == 0) {
+		return 0;
+	}
+
+	int i = 0, j = 0;
+	char rule[128] = {0};
+	char *network = NULL;
+	for (i=0; i<c->ports_num; i++) {
+		// setup port mapping only if host_port is set
+		if (c->ports[i].host_port > 0) {
+			for (j=0; j<pod->portmap_white_lists->e_num; j++) {
+				network = pod->portmap_white_lists->external_networks[j];
+
+				// redirect host_port to container_port
+				if (c->ports[i].host_port != c->ports[i].container_port) {
+					sprintf(rule, "-s %s -p %s -m %s --dport %d -j REDIRECT --to-ports %d",
+						network,
+						c->ports[i].protocol,
+						c->ports[i].protocol,
+						c->ports[i].host_port,
+						c->ports[i].container_port);
+					struct ipt_rule redirect_rule = {
+						.table = "nat",
+						.op = "-I",
+						.chain = "hyperstart-PREROUTING",
+						.rule = rule,
+					};
+					if (hyper_setup_iptables_rule(redirect_rule)<0) {
+						fprintf(stderr, "setup redirect_rule '%s' failed\n", rule);
+						return -1;
+					}
+				}
+
+				// open container_port to external network
+				sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
+					network,
+					c->ports[i].protocol,
+					c->ports[i].protocol,
+					c->ports[i].container_port);
+				struct ipt_rule accept_rule = {
+					.table = "filter",
+					.op = "-I",
+					.chain = "hyperstart-INPUT",
+					.rule = rule,
+				};
+				if (hyper_setup_iptables_rule(accept_rule)<0) {
+					fprintf(stderr, "setup accept_rule '%s' failed\n", rule);
+					return -1;
+				}
+			}
+		}
+
+		// only allow network request from white list
+		for (j=0; j<pod->portmap_white_lists->i_num; j++) {
+			sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
+				pod->portmap_white_lists->internal_networks[j],
+				c->ports[i].protocol,
+				c->ports[i].protocol,
+				c->ports[i].container_port);
+			struct ipt_rule accept_rule = {
+				.table = "filter",
+				.op = "-I",
+				.chain = "hyperstart-INPUT",
+				.rule = rule,
+			};
+			if (hyper_setup_iptables_rule(accept_rule)<0) {
+				fprintf(stderr, "setup accept_rule '%s' failed\n", rule);
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod)
+{
+	if (pod->portmap_white_lists == NULL || (pod->portmap_white_lists->i_num == 0 &&
+			pod->portmap_white_lists->e_num == 0)) {
+		return;
+	}
+
+	if (c->ports_num == 0) {
+		return;
+	}
+
+	int i = 0, j = 0;
+	char rule[128] = {0};
+	char *network = NULL;
+	for (i=0; i<c->ports_num; i++) {
+		// clean up port mapping only if host_port is set
+		if (c->ports[i].host_port > 0) {
+			for (j=0; j<pod->portmap_white_lists->e_num; j++) {
+				network = pod->portmap_white_lists->external_networks[j];
+
+				//  delete rules redirecting host_port to container_port
+				if (c->ports[i].host_port != c->ports[i].container_port) {
+					sprintf(rule, "-s %s -p %s -m %s --dport %d -j REDIRECT --to-ports %d",
+						network,
+						c->ports[i].protocol,
+						c->ports[i].protocol,
+						c->ports[i].host_port,
+						c->ports[i].container_port);
+					struct ipt_rule redirect_rule = {
+						.table = "nat",
+						.op = "-D",
+						.chain = "hyperstart-PREROUTING",
+						.rule = rule,
+					};
+					if (hyper_setup_iptables_rule(redirect_rule)<0) {
+						fprintf(stderr, "cleanup redirect '%s' failed\n", rule);
+					}
+				}
+
+				// open container_port to external network
+				sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
+					network,
+					c->ports[i].protocol,
+					c->ports[i].protocol,
+					c->ports[i].container_port);
+				struct ipt_rule accept_rule = {
+					.table = "filter",
+					.op = "-D",
+					.chain = "hyperstart-INPUT",
+					.rule = rule,
+				};
+				if (hyper_setup_iptables_rule(accept_rule)<0) {
+					fprintf(stderr, "cleanup accept_rule '%s' failed\n", rule);
+				}
+			}
+		}
+
+		for (j=0; j<pod->portmap_white_lists->i_num; j++) {
+			sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
+				pod->portmap_white_lists->internal_networks[j],
+				c->ports[i].protocol,
+				c->ports[i].protocol,
+				c->ports[i].container_port);
+			struct ipt_rule accept_rule = {
+				.table = "filter",
+				.op = "-D",
+				.chain = "hyperstart-INPUT",
+				.rule = rule,
+			};
+			if (hyper_setup_iptables_rule(accept_rule)<0) {
+				fprintf(stderr, "cleanup accept_rule '%s' failed\n", rule);
+			}
+		}
+	}
+}

src/portmapping.h

@@ -0,0 +1,23 @@
+#ifndef _PORT_MAPPING_H_
+#define _PORT_MAPPING_H_
+
+#include <stdio.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+struct ipt_rule {
+	char *table;
+	char *op;
+	char *chain;
+	char *rule;
+};
+
+struct hyper_pod;
+struct hyper_container;
+int hyper_setup_portmapping(struct hyper_pod *pod);
+void hyper_cleanup_portmapping(struct hyper_pod *pod);
+int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod);
+void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod);
+
+#endif

src/util.c

@@ -9,11 +9,15 @@
 #include <errno.h>
 #include <signal.h>
 #include <ctype.h>
+#include <unistd.h>
 #include <mntent.h>
+#include <sys/wait.h>
 #include <sys/mount.h>
 #include <sys/socket.h>
 #include <sys/reboot.h>
 #include <linux/reboot.h>
+#include <grp.h>
+#include <pwd.h>
 
 #include "util.h"
 #include "hyper.h"
@@ -64,13 +68,34 @@ int hyper_list_dir(char *path)
 	return 0;
 }
 
-int hyper_find_sd(char *prefix, char *addr, char **dev) {
+int hyper_copy_dir(char *src, char *dest)
+{
+	char cmd[512];
+	snprintf(cmd, sizeof(cmd), "/tar cf - -C %s . | /tar fx - -C %s", src, dest);
+
+	return hyper_cmd(cmd);
+}
+
+void hyper_sync_time_hctosys() {
+	int pid;
+	pid = fork();
+	if (pid < 0) {
+		perror("fail to fork to copy directory");
+	} else if (pid == 0) {
+		execlp("/busybox", "hwclock", "-s", NULL);
+		perror("exec hwclock -s command failed");
+		exit(-1);
+	}
+}
+
+int hyper_find_sd(char *addr, char **dev)
+{
 	struct dirent **list;
 	struct dirent *dir;
 	char path[512];
 	int i, num;
 
-	sprintf(path, "%s/sys/class/scsi_disk/0:0:%s/device/block/", prefix, addr);
+	sprintf(path, "/sys/class/scsi_disk/0:0:%s/device/block/", addr);
 	fprintf(stdout, "orig dev %s, scan path %s\n", *dev, path);
 
 	num = scandir(path, &list, NULL, NULL);
@@ -86,7 +111,6 @@ int hyper_find_sd(char *prefix, char *addr, char **dev) {
 		}
 
 		fprintf(stdout, "%s get %s\n", path, dir->d_name);
-		free(*dev);
 		*dev = strdup(dir->d_name);
 		break;
 	}
@@ -98,46 +122,290 @@ int hyper_find_sd(char *prefix, char *addr, char **dev) {
 	return 0;
 }
 
-int hyper_mkdir(char *hyper_path)
+static unsigned long id_or_max(const char *name)
+{
+	char *ptr;
+	long id = strtol(name, &ptr, 10);
+	if (name == ptr || id < 0 || (errno != 0 && id == 0) || *ptr != '\0')
+		return ~0UL;
+	return id;
+}
+
+// the same as getpwnam(), but it only parses /etc/passwd and allows name to be id string
+struct passwd *hyper_getpwnam(const char *name)
+{
+	uid_t uid = (uid_t)id_or_max(name);
+	FILE *file = fopen("/etc/passwd", "r");
+	if (!file) {
+		perror("faile to open /etc/passwd");
+		return NULL;
+	}
+	for (;;) {
+		struct passwd *pwd = fgetpwent(file);
+		if (!pwd)
+			break;
+		if (!strcmp(pwd->pw_name, name) || pwd->pw_uid == uid) {
+			fclose(file);
+			return pwd;
+		}
+	}
+	fclose(file);
+	return NULL;
+}
+
+// the same as getgrnam(), but it only parses /etc/group and allows the name to be id string
+struct group *hyper_getgrnam(const char *name)
+{
+	gid_t gid = (gid_t)id_or_max(name);
+	FILE *file = fopen("/etc/group", "r");
+	if (!file) {
+		perror("faile to open /etc/group");
+		return NULL;
+	}
+	for (;;) {
+		struct group *gr = fgetgrent(file);
+		if (!gr)
+			break;
+		if (!strcmp(gr->gr_name, name) || gr->gr_gid == gid) {
+			fclose(file);
+			return gr;
+		}
+	}
+	fclose(file);
+	return NULL;
+}
+
+// the same as getgrouplist(), but it only parses /etc/group
+int hyper_getgrouplist(const char *user, gid_t group, gid_t *groups, int *ngroups)
+{
+	int nr = 0, ret;
+	FILE *file = fopen("/etc/group", "r");
+	if (!file) {
+		perror("faile to open /etc/group");
+		return -1;
+	}
+	for (;;) {
+		struct group *gr = fgetgrent(file);
+		if (!gr)
+			break;
+		int j;
+		for (j = 0; gr->gr_mem && gr->gr_mem[j]; j++) {
+			if (!strcmp(gr->gr_mem[j], user)) {
+				if (nr + 1 < *ngroups)
+					groups[nr] = gr->gr_gid;
+				nr++;
+			}
+		}
+	}
+	fclose(file);
+	if (nr == 0) {
+		if (nr + 1 < *ngroups)
+			groups[nr] = group;
+		nr++;
+	}
+	ret = nr <= *ngroups ? nr : -1;
+	*ngroups = nr;
+	return ret;
+}
+
+int hyper_write_file(const char *path, const char *value, size_t len)
+{
+	size_t size = 0, l;
+	int fd = open(path, O_WRONLY);
+	if (fd < 0) {
+		perror("open file failed");
+		return -1;
+	}
+
+	while (size < len) {
+		l = write(fd, value + size, len - size);
+		if (l < 0) {
+			perror("fail to write to file");
+			close(fd);
+			return -1;
+		}
+		size += l;
+	}
+
+	close(fd);
+	return 0;
+}
+
+/* Trim all trailing '/' of a hyper_path except for the prefix one. */
+void hyper_filize(char *hyper_path)
+{
+	char *p;
+
+	if (strlen(hyper_path) == 0)
+		return;
+
+	p = &hyper_path[strlen(hyper_path) - 1];
+
+	for (; *p == '/' && p != hyper_path; p--) {
+		*p = '\0';
+	}
+}
+
+static int hyper_create_parent_dir(const char *hyper_path)
+{
+	char *p, *path = strdup(hyper_path);
+	int ret = 0;
+
+	if (path == NULL)
+		return -1;
+	p = strrchr(path, '/');
+	if (p != NULL && p != path) {
+		*p = '\0';
+		ret = hyper_mkdir(path, 0777);
+	}
+	free(path);
+
+	return ret;
+}
+
+/* hyper_path must point to a file rather than a directory, e.g., having trailing '/' */
+int hyper_create_file(const char *hyper_path)
+{
+	int fd;
+	struct stat stbuf;
+
+	if (stat(hyper_path, &stbuf) >= 0) {
+		if (S_ISREG(stbuf.st_mode))
+			return 0;
+		errno = S_ISDIR(stbuf.st_mode) ? EISDIR : EINVAL;
+		return -1;
+	}
+
+	if (hyper_create_parent_dir(hyper_path) < 0)
+		return -1;
+
+	fd = open(hyper_path, O_CREAT|O_WRONLY, 0666);
+	if (fd < 0)
+		return -1;
+	close(fd);
+	fprintf(stdout, "created file %s\n", hyper_path);
+	return 0;
+}
+
+int hyper_mkdir(char *hyper_path, mode_t mode)
 {
 	struct stat st;
 	char *p, *path = strdup(hyper_path);
 
 	if (path == NULL) {
 		errno = ENOMEM;
-		return -1;
+		goto fail;
 	}
 
-	fprintf(stdout, "create directory %s\n", path);
 	if (stat(path, &st) >= 0) {
 		if (S_ISDIR(st.st_mode))
-			return 0;
+			goto out;
 		errno = ENOTDIR;
-		return -1;
+		goto fail;
 	}
 
 	if (errno != ENOENT)
-		return -1;
+		goto fail;
 
 	p = strrchr(path, '/');
 	if (p == NULL) {
 		errno = EINVAL;
-		return -1;
+		goto fail;
 	}
 
 	if (p != path) {
 		*p = '\0';
 
-		if (hyper_mkdir(path) < 0)
-			return -1;
+		if (hyper_mkdir(path, mode) < 0)
+			goto fail;
 
 		*p = '/';
 	}
 
-	if (mkdir(path, 0755) < 0 && errno != EEXIST)
-		return -1;
-
+	fprintf(stdout, "create directory %s\n", path);
+	if (mkdir(path, mode) < 0 && errno != EEXIST) {
+		perror("failed to create directory");
+		goto fail;
+	}
+out:
+	free(path);
 	return 0;
+
+fail:
+	free(path);
+	return -1;
+}
+
+void online_cpu(void)
+{
+	DIR *dir = opendir("/sys/devices/system/cpu");
+	if (dir == NULL) {
+		fprintf(stderr, "open dir /sys/devices/system/cpu failed\n");
+		return;
+	}
+	printf("online_cpu()\n");
+	for (;;) {
+		int num;
+		int ret;
+		char path[256];
+		int fd;
+
+		struct dirent *entry = readdir(dir);
+		if (entry == NULL)
+			break;
+		if (entry->d_type != DT_DIR)
+			continue;
+		ret = sscanf(entry->d_name, "cpu%d", &num);
+		if (ret < 1 || num == 0) /* skip none cpu%d and cpu0 */
+			continue;
+		sprintf(path, "/sys/devices/system/cpu/%s/online", entry->d_name);
+		fd = open(path, O_RDWR);
+		if (fd < 0) {
+			fprintf(stderr, "open %s failed\n", path);
+			continue;
+		}
+		printf("try to online %s\n", entry->d_name);
+		ret = write(fd, "1", sizeof("1"));
+		printf("online %s result: %s\n", entry->d_name, ret == 2 ? "success" : "failed");
+		close(fd);
+	}
+	closedir(dir);
+}
+
+void online_memory(void)
+{
+	DIR *dir = opendir("/sys/devices/system/memory");
+	if (dir == NULL) {
+		fprintf(stderr, "open dir /sys/devices/system/memory failed\n");
+		return;
+	}
+	printf("online_memory()\n");
+	for (;;) {
+		int num;
+		int ret;
+		char path[256];
+		int fd;
+
+		struct dirent *entry = readdir(dir);
+		if (entry == NULL)
+			break;
+		if (entry->d_type != DT_DIR)
+			continue;
+		ret = sscanf(entry->d_name, "memory%d", &num);
+		if (ret < 1 || num == 0) /* skip none memory%d and memory0 */
+			continue;
+		sprintf(path, "/sys/devices/system/memory/%s/online", entry->d_name);
+		fd = open(path, O_RDWR);
+		if (fd < 0) {
+			fprintf(stderr, "open %s failed\n", path);
+			continue;
+		}
+		printf("try to online %s\n", entry->d_name);
+		ret = write(fd, "1", sizeof("1"));
+		printf("online %s result: %s\n", entry->d_name, ret == 2 ? "success" : "failed");
+		close(fd);
+	}
+	closedir(dir);
 }
 
 #if WITH_VBOX
@@ -358,21 +626,6 @@ int hyper_setfd_nonblock(int fd)
 	return flags;
 }
 
-int hyper_socketpair(int domain, int type, int protocol, int sv[2])
-{
-	if (socketpair(domain, type, protocol, sv) < 0) {
-		perror("socketpair failed");
-		return -1;
-	}
-
-	if (hyper_setfd_cloexec(sv[0]) < 0 ||
-	    hyper_setfd_cloexec(sv[1]) < 0) {
-		return -1;
-	}
-
-	return 0;
-}
-
 static void hyper_unmount_all(void)
 {
 	FILE *mtab;
@@ -413,14 +666,46 @@ static void hyper_unmount_all(void)
 			fprintf(stdout, ("umount %s: %s failed\n"),
 				filesys, strerror(errno));
 		}
+		free(filesys);
+		mntlist[i] = NULL;
 	}
 
 	sync();
 }
 
-void hyper_shutdown()
+void hyper_shutdown(int error)
 {
-	hyper_send_msg_block(ctl.chan.fd, ACK, 0, NULL);
+	hyper_send_msg_block(ctl.chan.fd, error?ERROR:ACK, 0, NULL);
 	hyper_unmount_all();
 	reboot(LINUX_REBOOT_CMD_POWER_OFF);
 }
+
+int hyper_cmd(char *cmd)
+{
+	int pid, status;
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fail to fork");
+		return -1;
+	} else if (pid > 0) {
+		if (waitpid(pid, &status, 0) <= 0) {
+			perror("waiting fork cmd failed");
+			return -1;
+		}
+		if (WIFEXITED(status)) {
+			int ret = WEXITSTATUS(status);
+			fprintf(stdout, "%s cmd exit normally, status %" PRIu8 "\n", cmd, ret);
+			if (ret == 0)
+				return 0;
+		}
+
+		fprintf(stdout, "cmd %s exit unexpectedly, status %" PRIu8 "\n", cmd, status);
+		return -1;
+	} else {
+		fprintf(stdout, "executing cmd %s\n", cmd);
+		execlp("/busybox", "sh", "-c", cmd, NULL);
+	}
+
+	return -1;
+}

src/util.h

@@ -2,6 +2,8 @@
 #define _UTIL_H_
 
 #include <stdio.h>
+#include <grp.h>
+#include <pwd.h>
 #include "../config.h"
 
 struct hyper_pod;
@@ -9,22 +11,33 @@ struct env;
 
 #ifdef WITH_DEBUG
 #define dprintf(fmt, ...) \
-	fprintf(stdout, fmt, ##_VA_ARGS__)
+	fprintf(stdout, fmt, ##__VA_ARGS__)
 #else
 #define dprintf(fmr, ...)
 #endif
 
 char *read_cmdline(void);
 int hyper_setup_env(struct env *envs, int num);
-int hyper_find_sd(char *prefix, char *addr, char **dev);
+int hyper_find_sd(char *addr, char **dev);
 int hyper_list_dir(char *path);
-int hyper_mkdir(char *path);
+int hyper_copy_dir(char *src, char *dst);
+void hyper_sync_time_hctosys();
+void online_cpu(void);
+void online_memory(void);
+int hyper_cmd(char *cmd);
+int hyper_create_file(const char *hyper_path);
+void hyper_filize(char *hyper_path);
+int hyper_mkdir(char *path, mode_t mode);
+int hyper_write_file(const char *path, const char *value, size_t len);
 int hyper_open_channel(char *channel, int mode);
 int hyper_open_serial_dev(char *tty);
 int hyper_setfd_cloexec(int fd);
 int hyper_setfd_block(int fd);
 int hyper_setfd_nonblock(int fd);
 int hyper_socketpair(int domain, int type, int protocol, int sv[2]);
-void hyper_shutdown(void);
+void hyper_shutdown(int ack);
 int hyper_insmod(char *module);
+struct passwd *hyper_getpwnam(const char *name);
+struct group *hyper_getgrnam(const char *name);
+int hyper_getgrouplist(const char *user, gid_t group, gid_t *groups, int *ngroups);
 #endif