From 00308d00b5fd9f55024f98c75853548bab4b2b8e Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 6 Jul 2016 11:39:27 +0800 Subject: [PATCH 1/5] split hyper_exec_cmd() Signed-off-by: Lai Jiangshan --- src/exec.c | 26 +++++++++++++++++--------- src/exec.h | 1 + 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/exec.c b/src/exec.c index dae3437..5b0b595 100644 --- a/src/exec.c +++ b/src/exec.c @@ -631,28 +631,38 @@ static void hyper_free_exec(struct hyper_exec *exec) int hyper_exec_cmd(char *json, int length) { struct hyper_exec *exec; - struct hyper_pod *pod = &global_pod; - int pipe[2] = {-1, -1}; - int pid, ret = -1; - uint32_t type; fprintf(stdout, "call hyper_exec_cmd, json %s, len %d\n", json, length); exec = hyper_parse_execcmd(json, length); if (exec == NULL) { fprintf(stderr, "parse exec cmd failed\n"); - goto out; + return -1; } + int ret = hyper_run_process(exec); + if (ret < 0) { + hyper_free_exec(exec); + } + return ret; +} + +int hyper_run_process(struct hyper_exec *exec) +{ + struct hyper_pod *pod = &global_pod; + int pipe[2] = {-1, -1}; + int pid, ret = -1; + uint32_t type; + if (exec->argv == NULL) { fprintf(stderr, "cmd is %p, seq %" PRIu64 ", container %s\n", exec->argv, exec->seq, exec->id); - goto free_exec; + goto out; } if (hyper_setup_exec_tty(exec) < 0) { fprintf(stderr, "setup exec tty failed\n"); - goto free_exec; + goto out; } if (hyper_watch_exec_pty(exec, pod) < 0) { @@ -699,8 +709,6 @@ close_tty: close(exec->stdinfd); close(exec->stdoutfd); close(exec->stderrfd); -free_exec: - hyper_free_exec(exec); goto out; } diff --git a/src/exec.h b/src/exec.h index f176262..54e35cf 100644 --- a/src/exec.h +++ b/src/exec.h @@ -49,6 +49,7 @@ int hyper_release_exec(struct hyper_exec *, struct hyper_pod *); int hyper_container_execcmd(struct hyper_pod *pod); int hyper_setup_exec_tty(struct hyper_exec *e); int hyper_dup_exec_tty(struct hyper_exec *e); +int hyper_run_process(struct hyper_exec *e); void hyper_exec_process(struct hyper_exec *e); struct hyper_exec *hyper_find_exec_by_pid(struct list_head *head, int pid); struct hyper_exec *hyper_find_exec_by_seq(struct hyper_pod *pod, uint64_t seq); From bb550c2444db183853104b9248eccc870de3af12 Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 6 Jul 2016 11:47:33 +0800 Subject: [PATCH 2/5] do not apply the container's env on exec let the hyperd do it. (merge the envs) Signed-off-by: Lai Jiangshan --- src/exec.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/exec.c b/src/exec.c index 5b0b595..96a6ce9 100644 --- a/src/exec.c +++ b/src/exec.c @@ -518,9 +518,8 @@ static int hyper_enter_container(struct hyper_pod *pod, /* TODO: wait for container finishing setup root */ chdir("/"); + ret = 0; - // TODO: let the hyperd do it (merging the env) when needed. - ret = hyper_setup_env(c->exec.envs, c->exec.envs_num); out: close(ipcns); close(utsns); From cd9a8f4306e7c3a3746422dc62a04069f5b83a58 Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 6 Jul 2016 13:53:05 +0800 Subject: [PATCH 3/5] set early env for exec Signed-off-by: Lai Jiangshan --- src/exec.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/exec.c b/src/exec.c index 96a6ce9..04a7822 100644 --- a/src/exec.c +++ b/src/exec.c @@ -563,6 +563,14 @@ static int hyper_do_exec_cmd(struct hyper_exec *exec, struct hyper_pod *pod, int goto exit; } + // set early env. the container env config can overwrite it + setenv("HOME", "/root", 1); + setenv("HOSTNAME", pod->hostname, 1); + if (exec->tty) + setenv("TERM", "xterm", 1); + else + unsetenv("TERM"); + hyper_exec_process(exec); exit: From 98b8211ddc1f30cc20d5780cbdde5895759e2f7c Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 6 Jul 2016 15:52:12 +0800 Subject: [PATCH 4/5] move hyper_start_containers() to hyper_start_pod() Signed-off-by: Lai Jiangshan --- src/init.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/init.c b/src/init.c index b40e686..27f22c8 100644 --- a/src/init.c +++ b/src/init.c @@ -414,7 +414,7 @@ int hyper_start_containers(struct hyper_pod *pod) return 0; } -static int hyper_setup_container(struct hyper_pod *pod) +static int hyper_setup_pod_init(struct hyper_pod *pod) { int stacksize = getpagesize() * 4; int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | @@ -461,11 +461,6 @@ static int hyper_setup_container(struct hyper_pod *pod) goto out; } - if (hyper_start_containers(pod) < 0) { - fprintf(stderr, "start containers failed\n"); - goto out; - } - ret = 0; out: close(arg.ctl_pipe[1]); @@ -587,7 +582,7 @@ static int hyper_setup_pod(struct hyper_pod *pod) return -1; } - if (hyper_setup_container(pod) < 0) { + if (hyper_setup_pod_init(pod) < 0) { fprintf(stderr, "start container failed\n"); return -1; } @@ -641,6 +636,12 @@ static int hyper_start_pod(char *json, int length) return -1; } + if (hyper_start_containers(pod) < 0) { + fprintf(stderr, "start containers failed\n"); + hyper_destroy_pod(pod); + return -1; + } + return 0; } From 54aed5efc5e071fe7c7e1baca2fe2e1c4e9e1da6 Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 6 Jul 2016 17:15:42 +0800 Subject: [PATCH 5/5] split container operation to hyper_setup_container() and hyper_run_process() Signed-off-by: Lai Jiangshan --- src/container.c | 69 ++++-------------------- src/container.h | 3 +- src/init.c | 139 ++++-------------------------------------------- 3 files changed, 19 insertions(+), 192 deletions(-) diff --git a/src/container.c b/src/container.c index 011e42f..ef7839c 100644 --- a/src/container.c +++ b/src/container.c @@ -499,47 +499,21 @@ static int hyper_rescan_scsi(void) struct hyper_container_arg { struct hyper_container *c; struct hyper_pod *pod; - int ipcns; - int utsns; int pipe[2]; }; -static int hyper_container_init(void *data) +static int hyper_setup_container_rootfs(void *data) { struct hyper_container_arg *arg = data; struct hyper_container *container = arg->c; char root[512], rootfs[512]; int setup_dns; - fprintf(stdout, "%s in\n", __func__); - if (container->exec.argv == NULL) { - fprintf(stdout, "no cmd!\n"); - goto fail; - } - - if (setns(arg->ipcns, CLONE_NEWIPC) < 0) { - perror("setns to ipcns of pod init faild"); - goto fail; - } - - if (setns(arg->utsns, CLONE_NEWUTS) < 0) { - perror("setns to ipcns of pod init faild"); - goto fail; - } - if (hyper_rescan_scsi() < 0) { fprintf(stdout, "rescan scsi failed\n"); goto fail; } - // set early env. the container env config can overwrite it - setenv("HOME", "/root", 1); - setenv("HOSTNAME", arg->pod->hostname, 1); - if (container->exec.tty) - setenv("TERM", "xterm", 1); - else - unsetenv("TERM"); - if (mount("", "/", NULL, MS_SLAVE|MS_REC, NULL) < 0) { perror("mount SLAVE failed"); goto fail; @@ -650,7 +624,8 @@ static int hyper_container_init(void *data) } hyper_send_type(arg->pipe[1], READY); - hyper_exec_process(&container->exec); + fflush(NULL); + _exit(0); fail: hyper_send_type(arg->pipe[1], ERROR); @@ -674,23 +649,15 @@ static int hyper_setup_pty(struct hyper_container *c) return -1; } - if (hyper_setup_exec_tty(&c->exec) < 0) { - fprintf(stderr, "setup container pts failed\n"); - return -1; - } - return 0; } -int hyper_start_container(struct hyper_container *container, - int utsns, int ipcns, struct hyper_pod *pod) +int hyper_setup_container(struct hyper_container *container, struct hyper_pod *pod) { int stacksize = getpagesize() * 42; struct hyper_container_arg arg = { .c = container, .pod = pod, - .utsns = utsns, - .ipcns = ipcns, .pipe = {-1, -1}, }; int flags = CLONE_NEWNS | SIGCHLD; @@ -699,24 +666,18 @@ int hyper_start_container(struct hyper_container *container, uint32_t type; int pid; - if (container->image == NULL || container->exec.argv == NULL) { - fprintf(stdout, "container root image %s, argv %p\n", - container->image, container->exec.argv); - goto fail; - } - if (pipe2(arg.pipe, O_CLOEXEC) < 0) { perror("create pipe between pod init execcmd failed"); goto fail; } - if (hyper_setup_pty(container) < 0) { - fprintf(stderr, "setup pty device for container failed\n"); + if (hyper_setup_container_portmapping(container, pod) < 0) { + perror("fail to setup port mapping for container"); goto fail; } - if (hyper_watch_exec_pty(&container->exec, pod) < 0) { - fprintf(stderr, "faile to watch container pty\n"); + if (hyper_setup_pty(container) < 0) { + fprintf(stderr, "setup pty device for container failed\n"); goto fail; } @@ -726,7 +687,7 @@ int hyper_start_container(struct hyper_container *container, goto fail; } - pid = clone(hyper_container_init, stack + stacksize, flags, &arg); + pid = clone(hyper_setup_container_rootfs, stack + stacksize, flags, &arg); free(stack); if (pid < 0) { perror("create child process failed"); @@ -746,24 +707,12 @@ int hyper_start_container(struct hyper_container *container, goto fail; } - container->exec.pid = pid; - list_add_tail(&container->exec.list, &pod->exec_head); - container->exec.ref++; - close(arg.pipe[0]); close(arg.pipe[1]); - fprintf(stdout, "container %s,init pid %d,ref %d\n", container->id, pid, container->exec.ref); return 0; fail: close(container->ns); - hyper_reset_event(&container->exec.stdinev); - hyper_reset_event(&container->exec.stdoutev); - hyper_reset_event(&container->exec.stderrev); container->ns = -1; - fprintf(stdout, "container %s init exit code %d\n", container->id, -1); - container->exec.code = -1; - container->exec.seq = 0; - container->exec.ref = 0; close(arg.pipe[0]); close(arg.pipe[1]); return -1; diff --git a/src/container.h b/src/container.h index ec7406c..19f1e5f 100644 --- a/src/container.h +++ b/src/container.h @@ -53,8 +53,7 @@ struct hyper_container { struct hyper_pod; -int hyper_start_container(struct hyper_container *container, - int utsns, int ipcns, struct hyper_pod *pod); +int hyper_setup_container(struct hyper_container *container, struct hyper_pod *pod); struct hyper_container *hyper_find_container(struct hyper_pod *pod, const char *id); void hyper_cleanup_container(struct hyper_container *container, struct hyper_pod *pod); void hyper_cleanup_containers(struct hyper_pod *pod); diff --git a/src/init.c b/src/init.c index 27f22c8..323b452 100644 --- a/src/init.c +++ b/src/init.c @@ -274,141 +274,18 @@ fail: goto out; } -struct hyper_stage0_arg { - struct hyper_pod *pod; - struct hyper_container *container; - int ctl_pipe[2]; -}; - -// stage0: enter the pidns -static int hyper_container_stage0(void *data) -{ - int pidns, ipcns, utsns, ret; - struct hyper_container *c; - struct hyper_stage0_arg *arg; - struct hyper_pod *pod; - char path[64]; - - arg = data; - pod = arg->pod; - c = arg->container; - - ret = pidns = ipcns = utsns = -1; - - sprintf(path, "/proc/%d/ns/pid", pod->init_pid); - pidns = open(path, O_RDONLY| O_CLOEXEC); - if (pidns < 0) { - perror("fail to open pidns of pod init"); - goto out; - } - - /* enter pidns of pod init, so the children of this process will run in - * pidns of pod init, see man 2 setns */ - if (setns(pidns, CLONE_NEWPID) < 0) { - perror("enter pidns of pod init failed"); - goto out; - } - - sprintf(path, "/proc/%d/ns/uts", pod->init_pid); - utsns = open(path, O_RDONLY| O_CLOEXEC); - if (utsns < 0) { - perror("fail to open utsns of pod init"); - goto out; - } - - sprintf(path, "/proc/%d/ns/ipc", pod->init_pid); - ipcns = open(path, O_RDONLY| O_CLOEXEC); - if (ipcns < 0) { - perror("fail to open ipcns of pod init"); - goto out; - } - - ret = hyper_start_container(c, utsns, ipcns, pod); -out: - close(pidns); - close(utsns); - close(ipcns); - - if (hyper_send_type(arg->ctl_pipe[1], ret ? ERROR : READY) < 0) { - fprintf(stderr, "container init send ready message failed\n"); - } - - /* hyper_container_stage0 shares fd table with init, let init closes pipe. */ - //close(arg->ctl_pipe[0]); - //close(arg->ctl_pipe[1]); - - _exit(ret); -} - -int hyper_start_container_stage0(struct hyper_container *c, struct hyper_pod *pod) -{ - int stacksize = getpagesize() * 4; - void *stack = NULL; - struct hyper_stage0_arg arg = { - .pod = pod, - .container = c, - .ctl_pipe = {-1, -1}, - }; - int ret = -1, pid, status; - uint32_t type; - - if (pipe2(arg.ctl_pipe, O_CLOEXEC) < 0) { - perror("create pipe between hyper init and pod init failed"); - goto out; - } - - stack = malloc(stacksize); - if (stack == NULL) { - perror("fail to allocate stack for container init"); - goto out; - } - - if (hyper_setup_container_portmapping(c, pod) < 0) { - perror("fail to setup port mapping for container"); - goto out; - } - - pid = clone(hyper_container_stage0, stack + stacksize, CLONE_VM| CLONE_FILES| SIGQUIT, &arg); - if (pid < 0) { - perror("enter container pid ns failed"); - goto out; - } - - if (waitpid(pid, &status, __WCLONE) <= 0) { - perror("waiting hyper_container_stage0 finish failed"); - goto out; - } - - /* Wait for container start */ - if (hyper_get_type(arg.ctl_pipe[0], &type) < 0) { - perror("get enter_container_pidns ready message failed"); - goto out; - } - - if (type != READY) { - fprintf(stderr, "get incorrect enter_container_pidns message type %d, expect READY\n", - type); - goto out; - } - - /* container process is spawned and ready to execute */ - pod->remains++; - ret = 0; -out: - close(arg.ctl_pipe[0]); - close(arg.ctl_pipe[1]); - - free(stack); - return ret; -} - int hyper_start_containers(struct hyper_pod *pod) { struct hyper_container *c; + // TODO: setup containers and run container init processes + // via separated hyperstart APIs list_for_each_entry(c, &pod->containers, list) { - if (hyper_start_container_stage0(c, pod) < 0) + if (hyper_setup_container(c, pod) < 0) return -1; + if (hyper_run_process(&c->exec) < 0) + return -1; + pod->remains++; } return 0; @@ -665,7 +542,9 @@ static int hyper_new_container(char *json, int length) } list_add_tail(&c->list, &pod->containers); - ret = hyper_start_container_stage0(c, pod); + ret = hyper_setup_container(c, pod); + if (ret >= 0) + ret = hyper_run_process(&c->exec); if (ret < 0) { //TODO full grace cleanup hyper_cleanup_container(c, pod);