diff --git a/build/iptables b/build/iptables new file mode 100755 index 0000000..2bb14c4 Binary files /dev/null and b/build/iptables differ diff --git a/build/libm.so.6 b/build/libm.so.6 new file mode 100755 index 0000000..9e471ce Binary files /dev/null and b/build/libm.so.6 differ diff --git a/build/make-initrd.sh b/build/make-initrd.sh index 48cb885..002356c 100755 --- a/build/make-initrd.sh +++ b/build/make-initrd.sh @@ -1,10 +1,12 @@ #!/bin/bash rm -rf root -mkdir -p root/lib +mkdir -p root/lib root/lib64 cp ../src/init ./root cp busybox ./root +cp iptables ./root +cp libm.so.6 ./root/lib64/ tar -xf modules.tar -C ./root/lib/ ldd ./root/init | while read line diff --git a/src/Makefile.am b/src/Makefile.am index 42acaef..08e6844 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,3 @@ AM_CFLAGS = -Wall bin_PROGRAMS=init -init_SOURCES=init.c jsmn.c net.c util.c parse.c container.c exec.c event.c +init_SOURCES=init.c jsmn.c net.c util.c parse.c container.c exec.c event.c portmapping.c diff --git a/src/container.c b/src/container.c index 3e610e9..2849188 100644 --- a/src/container.c +++ b/src/container.c @@ -768,7 +768,7 @@ struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id) return NULL; } -void hyper_cleanup_container(struct hyper_container *c) +void hyper_cleanup_container(struct hyper_container *c, struct hyper_pod *pod) { char root[512]; @@ -777,6 +777,7 @@ void hyper_cleanup_container(struct hyper_container *c) perror("umount devpts failed"); close(c->ns); + hyper_cleanup_container_portmapping(c, pod); hyper_free_container(c); } @@ -785,7 +786,7 @@ void hyper_cleanup_containers(struct hyper_pod *pod) struct hyper_container *c, *n; list_for_each_entry_safe(c, n, &pod->containers, list) - hyper_cleanup_container(c); + hyper_cleanup_container(c, pod); pod->remains = 0; } diff --git a/src/container.h b/src/container.h index 0b7bddb..86cf459 100644 --- a/src/container.h +++ b/src/container.h @@ -24,6 +24,12 @@ struct sysctl { char *value; }; +struct port { + int host_port; + int container_port; + char *protocol; +}; + struct hyper_container { char *id; char *rootfs; @@ -33,9 +39,11 @@ struct hyper_container { struct volume *vols; struct fsmap *maps; struct sysctl *sys; + struct port *ports; int vols_num; int maps_num; int sys_num; + int ports_num; int ns; int initialize; uint32_t code; @@ -48,7 +56,7 @@ struct hyper_pod; int hyper_start_container(struct hyper_container *container, int utsns, int ipcns, struct hyper_pod *pod); struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id); -void hyper_cleanup_container(struct hyper_container *container); +void hyper_cleanup_container(struct hyper_container *container, struct hyper_pod *pod); void hyper_cleanup_containers(struct hyper_pod *pod); void hyper_free_container(struct hyper_container *c); diff --git a/src/hyper.h b/src/hyper.h index 51e5471..26d8685 100644 --- a/src/hyper.h +++ b/src/hyper.h @@ -12,6 +12,7 @@ #include "exec.h" #include "event.h" #include "container.h" +#include "portmapping.h" enum { RESERVED, @@ -49,6 +50,7 @@ struct hyper_pod { struct hyper_interface *iface; struct hyper_route *rt; char **dns; + char **white_cidrs; struct list_head containers; struct list_head exec_head; char *hostname; @@ -58,6 +60,7 @@ struct hyper_pod { uint32_t r_num; uint32_t e_num; uint32_t d_num; + uint32_t w_num; uint32_t type; /* how many containers are running */ uint32_t remains; diff --git a/src/init.c b/src/init.c index ef58481..6fb6e8c 100644 --- a/src/init.c +++ b/src/init.c @@ -363,6 +363,11 @@ int hyper_start_container_stage0(struct hyper_container *c, struct hyper_pod *po goto out; } + if (hyper_setup_container_portmapping(c, pod) < 0) { + perror("fail to setup port mapping for container"); + goto out; + } + pid = clone(hyper_container_stage0, stack + stacksize, CLONE_VM| CLONE_FILES| SIGQUIT, &arg); if (pid < 0) { perror("enter container pid ns failed"); @@ -577,6 +582,11 @@ static int hyper_setup_pod(struct hyper_pod *pod) return -1; } + if (hyper_setup_portmapping(pod) < 0) { + fprintf(stderr, "setup port mapping failed\n"); + return -1; + } + if (hyper_setup_container(pod) < 0) { fprintf(stderr, "start container failed\n"); return -1; @@ -657,7 +667,7 @@ static int hyper_new_container(char *json, int length) ret = hyper_start_container_stage0(c, pod); if (ret < 0) { //TODO full grace cleanup - hyper_cleanup_container(c); + hyper_cleanup_container(c, pod); } return ret; @@ -960,6 +970,7 @@ void hyper_cleanup_pod(struct hyper_pod *pod) hyper_cleanup_network(pod); hyper_cleanup_shared(pod); hyper_cleanup_dns(pod); + hyper_cleanup_portmapping(pod); hyper_cleanup_hostname(pod); } @@ -1282,6 +1293,10 @@ int main(int argc, char *argv[]) symlink("/busybox", "/sh"); symlink("/busybox", "/tar"); symlink("/busybox", "/sbin/modprobe"); + symlink("/busybox", "/sbin/depmod"); + symlink("/iptables", "/sbin/iptables"); + symlink("/iptables", "/sbin/iptables-restore"); + symlink("/iptables", "/sbin/iptables-save"); cmdline = read_cmdline(); diff --git a/src/parse.c b/src/parse.c index 0ccd337..2f7b7bc 100644 --- a/src/parse.c +++ b/src/parse.c @@ -524,6 +524,72 @@ static int hyper_parse_process(struct hyper_exec *exec, char *json, jsmntok_t *t return i; } +static void container_free_ports(struct hyper_container *c) +{ + int i; + + for (i = 0; i < c->ports_num; i++) { + free(c->ports[i].protocol); + } + free(c->ports); + c->ports = NULL; + c->ports_num = 0; +} + +static int container_parse_ports(struct hyper_container *c, char *json, jsmntok_t *toks) +{ + int i = 0, j; + + if (toks[i].size == 0) { + return 0; + } + + if (toks[i].type != JSMN_ARRAY) { + fprintf(stdout, "ports format error\n"); + return -1; + } + + c->ports = calloc(toks[i].size, sizeof(*c->ports)); + if (c->ports == NULL) { + fprintf(stderr, "allocate memory for ports failed\n"); + return -1; + } + + c->ports_num = toks[i].size; + fprintf(stdout, "ports num %d\n", c->ports_num); + + i++; + for (j = 0; j < c->ports_num; j++) { + int i_port, next_port; + + if (toks[i].type != JSMN_OBJECT) { + fprintf(stdout, "port array need object\n"); + return -1; + } + next_port = toks[i].size; + i++; + for (i_port = 0; i_port < next_port; i_port++, i++) { + if (json_token_streq(json, &toks[i], "protocol")) { + c->ports[j].protocol = + (json_token_str(json, &toks[++i])); + fprintf(stdout, "port %d protocol %s\n", j, c->ports[j].protocol); + } else if (json_token_streq(json, &toks[i], "hostPort")) { + c->ports[j].host_port = json_token_int(json, &toks[++i]); + fprintf(stdout, "port %d host_port %d\n", j, c->ports[j].host_port); + } else if (json_token_streq(json, &toks[i], "containerPort")) { + c->ports[j].container_port = json_token_int(json, &toks[++i]); + fprintf(stdout, "port %d container_port %d\n", j, c->ports[j].container_port); + } else { + fprintf(stdout, "get unknown section %s in ports\n", + json_token_str(json, &toks[i])); + return -1; + } + } + } + + return i; +} + void hyper_free_container(struct hyper_container *c) { free(c->id); @@ -542,6 +608,7 @@ void hyper_free_container(struct hyper_container *c) c->fstype = NULL; container_free_volumes(c); + container_free_ports(c); container_free_sysctl(c); container_free_fsmap(c); container_cleanup_exec(&c->exec); @@ -636,6 +703,11 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container * fprintf(stdout, "need to initialize container\n"); } i++; + } else if (json_token_streq(json, t, "ports") && t->size == 1) { + next = container_parse_ports(c, json, &toks[++i]); + if (next < 0) + goto fail; + i += next; } else { fprintf(stdout, "get unknown section %s in container\n", json_token_str(json, t)); @@ -931,6 +1003,37 @@ static int hyper_parse_dns(struct hyper_pod *pod, char *json, jsmntok_t *toks) return i; } +static int hyper_parse_white_cidrs(struct hyper_pod *pod, char *json, jsmntok_t *toks) +{ + int i = 0, j; + + if (toks[i].size == 0) { + return 0; + } + + if (toks[i].type != JSMN_ARRAY) { + fprintf(stdout, "white CIDRs format incorrect\n"); + return -1; + } + + pod->w_num = toks[i].size; + fprintf(stdout, "white CIDRs count %d\n", pod->w_num); + + pod->white_cidrs = calloc(pod->w_num, sizeof(*pod->white_cidrs)); + if (pod->white_cidrs == NULL) { + fprintf(stdout, "alloc memory for white_cidrs failed\n"); + return -1; + } + + i++; + for (j = 0; j < pod->w_num; j++, i++) { + pod->white_cidrs[j] = (json_token_str(json, &toks[i])); + fprintf(stdout, "pod white_cidr %d: %s\n", j, pod->white_cidrs[j]); + } + + return i; +} + int hyper_parse_pod(struct hyper_pod *pod, char *json, int length) { int i, n, next = -1; @@ -1012,6 +1115,12 @@ realloc: pod->policy = POLICY_ONFAILURE; fprintf(stdout, "restartPolicy is %" PRIu8 "\n", pod->policy); i++; + } else if (json_token_streq(json, t, "whiteCIDRs") && t->size == 1) { + next = hyper_parse_white_cidrs(pod, json, &toks[++i]); + if (next < 0) + goto out; + + i += next; } else { fprintf(stdout, "get unknown section %s in pod\n", json_token_str(json, &toks[i])); diff --git a/src/portmapping.c b/src/portmapping.c new file mode 100644 index 0000000..acce872 --- /dev/null +++ b/src/portmapping.c @@ -0,0 +1,348 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hyper.h" +#include "util.h" +#include "../config.h" + +int hyper_init_modules() +{ + int status = hyper_cmd("/sbin/depmod"); + if (status != 0) { + fprintf(stderr, "depmod failed, status: %d\n", status); + return -1; + } + + return 0; +} + +int hyper_setup_iptables_rule(struct ipt_rule rule) +{ + char check_cmd[512] = {0}; + char cmd[512] = {0}; + int check = -1; + + if (rule.rule != NULL) { + sprintf(check_cmd, "/sbin/iptables -t %s -C %s %s", rule.table, rule.chain, rule.rule); + sprintf(cmd, "/sbin/iptables -t %s %s %s %s", rule.table, rule.op, rule.chain, rule.rule); + } else { + sprintf(cmd, "/sbin/iptables -t %s %s %s", rule.table, rule.op, rule.chain); + } + + if (strlen(check_cmd) > 0) { + check = hyper_cmd(check_cmd); + fprintf(stdout, "check iptables '%s', ret: %d\n", check_cmd, check); + } + + if (check == 0) { + // iptables rule already exist, do not insert it again + if (!strncmp(rule.op, "-A", strlen("-A")) || + !strncmp(rule.op, "-I", strlen("-I")) || + !strncmp(rule.op, "-N", strlen("-N"))) { + fprintf(stdout, "iptables rule '%s' already exist\n", rule.rule); + return 0; + } + } + + int status = hyper_cmd(cmd); + fprintf(stdout, "insert iptables '%s', ret: %d\n", cmd, status); + if (status != 0) { + fprintf(stderr, "insert iptables rule failed, ret: %d\n", status); + return -1; + } + + return 0; +} + +// initialize modules and iptables chains +int hyper_setup_portmapping(struct hyper_pod *pod) +{ + if (pod->w_num == 0) { + return 0; + } + + if (hyper_init_modules() < 0) { + return -1; + } + + // iptables -t filter -N hyperstart-INPUT + // iptables -t nat -N hyperstart-PREROUTING + // iptables -t filter -I INPUT -j hyperstart-INPUT + // iptables -t nat -I PREROUTING -j hyperstart-PREROUTING + // iptables -t filter -A hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -p icmp -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -i lo -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -j DROP + // iptables -t nat -A hyperstart-PREROUTING -j RETURN + const struct ipt_rule rules[] = { + { + .table = "filter", + .op = "-N", + .chain = "hyperstart-INPUT", + .rule = NULL, + }, + { + .table = "nat", + .op = "-N", + .chain = "hyperstart-PREROUTING", + .rule = NULL, + }, + { + .table = "filter", + .op = "-I", + .chain = "INPUT", + .rule = "-j hyperstart-INPUT", + }, + { + .table = "nat", + .op = "-I", + .chain = "PREROUTING", + .rule = "-j hyperstart-PREROUTING", + }, + { + .table = "filter", + .op = "-A", + .chain = "hyperstart-INPUT", + .rule = "-m state --state RELATED,ESTABLISHED -j ACCEPT", + }, + { + .table = "filter", + .op = "-A", + .chain = "hyperstart-INPUT", + .rule = "-p icmp -j ACCEPT", + }, + { + .table = "filter", + .op = "-A", + .chain = "hyperstart-INPUT", + .rule = "-i lo -j ACCEPT", + }, + { + .table = "filter", + .op = "-A", + .chain = "hyperstart-INPUT", + .rule = "-j DROP", + }, + { + .table = "nat", + .op = "-A", + .chain = "hyperstart-PREROUTING", + .rule = "-j RETURN", + }, + }; + + int i = 0; + for(i=0; i< sizeof(rules)/sizeof(struct ipt_rule); i++) { + if (hyper_setup_iptables_rule(rules[i])<0) { + return -1; + } + } + + return 0; +} + +void hyper_cleanup_portmapping(struct hyper_pod *pod) +{ + if (pod->w_num == 0) { + return; + } + + // iptables -t filter -D hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + // iptables -t filter -D hyperstart-INPUT -p icmp -j ACCEPT + // iptables -t filter -D hyperstart-INPUT -i lo -j ACCEPT + // iptables -t filter -D hyperstart-INPUT -j DROP + // iptables -t filter -D INPUT -j hyperstart-DNPUT + // iptables -t nat -D hyperstart-PREROUTING -j RETURN + // iptables -t nat -D PREROUTING -j hyperstart-PREROUTING + // iptables -t filter -F hyperstart-INPUT + // iptables -t nat -F hyperstart-PREROUTING + // iptables -t filter -X hyperstart-INPUT + // iptables -t nat -X hyperstart-PREROUTING + const struct ipt_rule rules[] = { + { + .table = "filter", + .op = "-D", + .chain = "hyperstart-INPUT", + .rule = "-m state --state RELATED,ESTABLISHED -j ACCEPT", + }, + { + .table = "filter", + .op = "-D", + .chain = "hyperstart-INPUT", + .rule = "-p icmp -j ACCEPT", + }, + { + .table = "filter", + .op = "-D", + .chain = "hyperstart-INPUT", + .rule = "-i lo -j ACCEPT", + }, + { + .table = "filter", + .op = "-D", + .chain = "hyperstart-INPUT", + .rule = "-j DROP", + }, + { + .table = "nat", + .op = "-D", + .chain = "hyperstart-PREROUTING", + .rule = "-j RETURN", + }, + { + .table = "nat", + .op = "-D", + .chain = "PREROUTING", + .rule = "-j hyperstart-PREROUTING", + }, + { + .table = "filter", + .op = "-D", + .chain = "INPUT", + .rule = "-j hyperstart-INPUT", + }, + { + .table = "nat", + .op = "-F", + .chain = "hyperstart-PREROUTING", + .rule = NULL, + }, + { + .table = "nat", + .op = "-X", + .chain = "hyperstart-PREROUTING", + .rule = NULL, + }, + { + .table = "filter", + .op = "-F", + .chain = "hyperstart-INPUT", + .rule = NULL, + }, + { + .table = "filter", + .op = "-X", + .chain = "hyperstart-INPUT", + .rule = NULL, + }, + }; + + int i = 0; + for(i=0; i< sizeof(rules)/sizeof(struct ipt_rule); i++) { + if (hyper_setup_iptables_rule(rules[i])<0) { + return; + } + } +} + +int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod) +{ + if (pod->w_num == 0) { + return 0; + } + + if (c->ports_num == 0) { + return 0; + } + + int i = 0, j = 0; + char rule[128] = {0}; + + for (i=0; iports_num; i++) { + sprintf(rule, "-p %s -m %s --dport %d -j REDIRECT --to-ports %d", + c->ports[i].protocol, + c->ports[i].protocol, + c->ports[i].host_port, + c->ports[i].container_port); + struct ipt_rule rediect_rule = { + .table = "nat", + .op = "-I", + .chain = "hyperstart-PREROUTING", + .rule = rule, + }; + if (hyper_setup_iptables_rule(rediect_rule)<0) { + fprintf(stderr, "setup rediect_rule '%s' failed\n", rule); + return -1; + } + + for (j=0; jw_num; j++) { + sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT", + pod->white_cidrs[j], + c->ports[i].protocol, + c->ports[i].protocol, + c->ports[i].container_port); + struct ipt_rule accept_rule = { + .table = "filter", + .op = "-I", + .chain = "hyperstart-INPUT", + .rule = rule, + }; + if (hyper_setup_iptables_rule(accept_rule)<0) { + fprintf(stderr, "setup accept_rule '%s' failed\n", rule); + return -1; + } + } + + } + + return 0; +} + +void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod) +{ + if (pod->w_num == 0) { + return; + } + + if (c->ports_num == 0) { + return; + } + + + int i = 0, j = 0; + char rule[128] = {0}; + + for (i=0; iports_num; i++) { + sprintf(rule, "-p %s -m %s --dport %d -j REDIRECT --to-ports %d", + c->ports[i].protocol, + c->ports[i].protocol, + c->ports[i].host_port, + c->ports[i].container_port); + struct ipt_rule rediect_rule = { + .table = "nat", + .op = "-D", + .chain = "hyperstart-PREROUTING", + .rule = rule, + }; + if (hyper_setup_iptables_rule(rediect_rule)<0) { + fprintf(stderr, "setup rediect_rule '%s' failed\n", rule); + } + + for (j=0; jw_num; j++) { + sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT", + pod->white_cidrs[j], + c->ports[i].protocol, + c->ports[i].protocol, + c->ports[i].container_port); + struct ipt_rule accept_rule = { + .table = "filter", + .op = "-D", + .chain = "hyperstart-INPUT", + .rule = rule, + }; + if (hyper_setup_iptables_rule(accept_rule)<0) { + fprintf(stderr, "setup accept_rule '%s' failed\n", rule); + } + } + + } +} diff --git a/src/portmapping.h b/src/portmapping.h new file mode 100644 index 0000000..453605a --- /dev/null +++ b/src/portmapping.h @@ -0,0 +1,23 @@ +#ifndef _PORT_MAPPING_H_ +#define _PORT_MAPPING_H_ + +#include +#include +#include +#include + +struct ipt_rule { + char *table; + char *op; + char *chain; + char *rule; +}; + +struct hyper_pod; +struct hyper_container; +int hyper_setup_portmapping(struct hyper_pod *pod); +void hyper_cleanup_portmapping(struct hyper_pod *pod); +int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod); +void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod); + +#endif diff --git a/src/util.c b/src/util.c index 6d1a278..90acf03 100644 --- a/src/util.c +++ b/src/util.c @@ -625,3 +625,33 @@ void hyper_shutdown() hyper_unmount_all(); reboot(LINUX_REBOOT_CMD_POWER_OFF); } + +int hyper_cmd(char *cmd) +{ + int pid, status; + + pid = fork(); + if (pid < 0) { + perror("fail to fork"); + return -1; + } else if (pid > 0) { + if (waitpid(pid, &status, 0) <= 0) { + perror("waiting fork cmd failed"); + return -1; + } + if (WIFEXITED(status)) { + int ret = WEXITSTATUS(status); + fprintf(stdout, "%s cmd exit normally, status %" PRIu8 "\n", cmd, ret); + if (ret == 0) + return 0; + } + + fprintf(stdout, "cmd %s exit unexpectedly, status %" PRIu8 "\n", cmd, status); + return -1; + } else { + fprintf(stdout, "executing cmd %s\n", cmd); + execlp("/busybox", "sh", "-c", cmd, NULL); + } + + return -1; +} diff --git a/src/util.h b/src/util.h index c81ff60..ed72a88 100644 --- a/src/util.h +++ b/src/util.h @@ -24,6 +24,7 @@ int hyper_copy_dir(char *src, char *dst); void hyper_sync_time_hctosys(); void online_cpu(void); void online_memory(void); +int hyper_cmd(char *cmd); int hyper_mkdir(char *path); int hyper_open_channel(char *channel, int mode); int hyper_open_serial_dev(char *tty);