Merge pull request #100 from feiskyer/port-mapping

Add port mapping

build/make-initrd.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 
 rm -rf root
-mkdir -p root/lib
+mkdir -p root/lib root/lib64
 
 cp ../src/init ./root
 cp busybox ./root
+cp iptables ./root
+cp libm.so.6 ./root/lib64/
 tar -xf modules.tar -C ./root/lib/
 
 ldd ./root/init | while read line

src/Makefile.am

@@ -1,3 +1,3 @@
 AM_CFLAGS = -Wall
 bin_PROGRAMS=init
-init_SOURCES=init.c jsmn.c net.c util.c parse.c container.c exec.c event.c
+init_SOURCES=init.c jsmn.c net.c util.c parse.c container.c exec.c event.c portmapping.c

src/container.c

@@ -768,7 +768,7 @@ struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id)
 	return NULL;
 }
 
-void hyper_cleanup_container(struct hyper_container *c)
+void hyper_cleanup_container(struct hyper_container *c, struct hyper_pod *pod)
 {
 	char root[512];
 
@@ -777,6 +777,7 @@ void hyper_cleanup_container(struct hyper_container *c)
 		perror("umount devpts failed");
 
 	close(c->ns);
+	hyper_cleanup_container_portmapping(c, pod);
 	hyper_free_container(c);
 }
 
@@ -785,7 +786,7 @@ void hyper_cleanup_containers(struct hyper_pod *pod)
 	struct hyper_container *c, *n;
 
 	list_for_each_entry_safe(c, n, &pod->containers, list)
-		hyper_cleanup_container(c);
+		hyper_cleanup_container(c, pod);
 
 	pod->remains = 0;
 }

src/container.h

@@ -24,6 +24,12 @@ struct sysctl {
 	char	*value;
 };
 
+struct port {
+	int  host_port;
+	int  container_port;
+	char *protocol;
+};
+
 struct hyper_container {
 	char			*id;
 	char			*rootfs;
@@ -33,9 +39,11 @@ struct hyper_container {
 	struct volume		*vols;
 	struct fsmap		*maps;
 	struct sysctl		*sys;
+	struct port			*ports;
 	int			vols_num;
 	int			maps_num;
 	int			sys_num;
+	int 		ports_num;
 	int			ns;
 	int			initialize;
 	uint32_t		code;
@@ -48,7 +56,7 @@ struct hyper_pod;
 int hyper_start_container(struct hyper_container *container,
 			  int utsns, int ipcns, struct hyper_pod *pod);
 struct hyper_container *hyper_find_container(struct hyper_pod *pod, char *id);
-void hyper_cleanup_container(struct hyper_container *container);
+void hyper_cleanup_container(struct hyper_container *container, struct hyper_pod *pod);
 void hyper_cleanup_containers(struct hyper_pod *pod);
 void hyper_free_container(struct hyper_container *c);
 

src/hyper.h

@@ -12,6 +12,7 @@
 #include "exec.h"
 #include "event.h"
 #include "container.h"
+#include "portmapping.h"
 
 enum {
 	RESERVED,
@@ -49,6 +50,7 @@ struct hyper_pod {
 	struct hyper_interface	*iface;
 	struct hyper_route	*rt;
 	char			**dns;
+	char 			**white_cidrs;
 	struct list_head	containers;
 	struct list_head	exec_head;
 	char			*hostname;
@@ -58,6 +60,7 @@ struct hyper_pod {
 	uint32_t		r_num;
 	uint32_t		e_num;
 	uint32_t		d_num;
+	uint32_t 		w_num;
 	uint32_t		type;
 	/* how many containers are running */
 	uint32_t		remains;

src/init.c

@@ -363,6 +363,11 @@ int hyper_start_container_stage0(struct hyper_container *c, struct hyper_pod *po
 		goto out;
 	}
 
+	if (hyper_setup_container_portmapping(c, pod) < 0) {
+		perror("fail to setup port mapping for container");
+		goto out;
+	}
+
 	pid = clone(hyper_container_stage0, stack + stacksize, CLONE_VM| CLONE_FILES| SIGQUIT, &arg);
 	if (pid < 0) {
 		perror("enter container pid ns failed");
@@ -577,6 +582,11 @@ static int hyper_setup_pod(struct hyper_pod *pod)
 		return -1;
 	}
 
+	if (hyper_setup_portmapping(pod) < 0) {
+		fprintf(stderr, "setup port mapping failed\n");
+		return -1;
+	}
+
 	if (hyper_setup_container(pod) < 0) {
 		fprintf(stderr, "start container failed\n");
 		return -1;
@@ -657,7 +667,7 @@ static int hyper_new_container(char *json, int length)
 	ret = hyper_start_container_stage0(c, pod);
 	if (ret < 0) {
 		//TODO full grace cleanup
-		hyper_cleanup_container(c);
+		hyper_cleanup_container(c, pod);
 	}
 
 	return ret;
@@ -960,6 +970,7 @@ void hyper_cleanup_pod(struct hyper_pod *pod)
 	hyper_cleanup_network(pod);
 	hyper_cleanup_shared(pod);
 	hyper_cleanup_dns(pod);
+	hyper_cleanup_portmapping(pod);
 	hyper_cleanup_hostname(pod);
 }
 
@@ -1282,6 +1293,10 @@ int main(int argc, char *argv[])
 	symlink("/busybox", "/sh");
 	symlink("/busybox", "/tar");
 	symlink("/busybox", "/sbin/modprobe");
+	symlink("/busybox", "/sbin/depmod");
+	symlink("/iptables", "/sbin/iptables");
+	symlink("/iptables", "/sbin/iptables-restore");
+	symlink("/iptables", "/sbin/iptables-save");
 
 	cmdline = read_cmdline();
 

src/parse.c

@@ -524,6 +524,72 @@ static int hyper_parse_process(struct hyper_exec *exec, char *json, jsmntok_t *t
 	return i;
 }
 
+static void container_free_ports(struct hyper_container *c)
+{
+	int i;
+
+	for (i = 0; i < c->ports_num; i++) {
+		free(c->ports[i].protocol);
+	}
+	free(c->ports);
+	c->ports = NULL;
+	c->ports_num = 0;
+}
+
+static int container_parse_ports(struct hyper_container *c, char *json, jsmntok_t *toks)
+{
+	int i = 0, j;
+
+	if (toks[i].size == 0) {
+		return 0;
+	}
+
+	if (toks[i].type != JSMN_ARRAY) {
+		fprintf(stdout, "ports format error\n");
+		return -1;
+	}
+
+	c->ports = calloc(toks[i].size, sizeof(*c->ports));
+	if (c->ports == NULL) {
+		fprintf(stderr, "allocate memory for ports failed\n");
+		return -1;
+	}
+
+	c->ports_num = toks[i].size;
+	fprintf(stdout, "ports num %d\n", c->ports_num);
+
+	i++;
+	for (j = 0; j < c->ports_num; j++) {
+		int i_port, next_port;
+
+		if (toks[i].type != JSMN_OBJECT) {
+			fprintf(stdout, "port array need object\n");
+			return -1;
+		}
+		next_port = toks[i].size;
+		i++;
+		for (i_port = 0; i_port < next_port; i_port++, i++) {
+			if (json_token_streq(json, &toks[i], "protocol")) {
+				c->ports[j].protocol =
+				(json_token_str(json, &toks[++i]));
+				fprintf(stdout, "port %d protocol %s\n", j, c->ports[j].protocol);
+			} else if (json_token_streq(json, &toks[i], "hostPort")) {
+				c->ports[j].host_port = json_token_int(json, &toks[++i]);
+				fprintf(stdout, "port %d host_port %d\n", j, c->ports[j].host_port);
+			} else if (json_token_streq(json, &toks[i], "containerPort")) {
+				c->ports[j].container_port = json_token_int(json, &toks[++i]);
+				fprintf(stdout, "port %d container_port %d\n", j, c->ports[j].container_port);
+			} else {
+				fprintf(stdout, "get unknown section %s in ports\n",
+					json_token_str(json, &toks[i]));
+				return -1;
+			}
+		}
+	}
+
+	return i;
+}
+
 void hyper_free_container(struct hyper_container *c)
 {
 	free(c->id);
@@ -542,6 +608,7 @@ void hyper_free_container(struct hyper_container *c)
 	c->fstype = NULL;
 
 	container_free_volumes(c);
+	container_free_ports(c);
 	container_free_sysctl(c);
 	container_free_fsmap(c);
 	container_cleanup_exec(&c->exec);
@@ -636,6 +703,11 @@ static int hyper_parse_container(struct hyper_pod *pod, struct hyper_container *
 				fprintf(stdout, "need to initialize container\n");
 			}
 			i++;
+		} else if (json_token_streq(json, t, "ports") && t->size == 1) {
+			next = container_parse_ports(c, json, &toks[++i]);
+			if (next < 0)
+				goto fail;
+			i += next;
 		} else {
 			fprintf(stdout, "get unknown section %s in container\n",
 				json_token_str(json, t));
@@ -931,6 +1003,37 @@ static int hyper_parse_dns(struct hyper_pod *pod, char *json, jsmntok_t *toks)
 	return i;
 }
 
+static int hyper_parse_white_cidrs(struct hyper_pod *pod, char *json, jsmntok_t *toks)
+{
+	int i = 0, j;
+
+	if (toks[i].size == 0) {
+		return 0;
+	}
+
+	if (toks[i].type != JSMN_ARRAY) {
+		fprintf(stdout, "white CIDRs format incorrect\n");
+		return -1;
+	}
+
+	pod->w_num = toks[i].size;
+	fprintf(stdout, "white CIDRs count %d\n", pod->w_num);
+
+	pod->white_cidrs = calloc(pod->w_num, sizeof(*pod->white_cidrs));
+	if (pod->white_cidrs == NULL) {
+		fprintf(stdout, "alloc memory for white_cidrs failed\n");
+		return -1;
+	}
+
+	i++;
+	for (j = 0; j < pod->w_num; j++, i++) {
+		pod->white_cidrs[j] = (json_token_str(json, &toks[i]));
+		fprintf(stdout, "pod white_cidr %d: %s\n", j, pod->white_cidrs[j]);
+	}
+
+	return i;
+}
+
 int hyper_parse_pod(struct hyper_pod *pod, char *json, int length)
 {
 	int i, n, next = -1;
@@ -1012,6 +1115,12 @@ realloc:
 				pod->policy = POLICY_ONFAILURE;
 			fprintf(stdout, "restartPolicy is %" PRIu8 "\n", pod->policy);
 			i++;
+		} else if (json_token_streq(json, t, "whiteCIDRs") && t->size == 1) {
+			next = hyper_parse_white_cidrs(pod, json, &toks[++i]);
+			if (next < 0)
+				goto out;
+
+			i += next;
 		} else {
 			fprintf(stdout, "get unknown section %s in pod\n",
 				json_token_str(json, &toks[i]));

src/portmapping.c

@@ -0,0 +1,348 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <limits.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/wait.h>
+#include <sys/utsname.h>
+#include <arpa/inet.h>
+#include <fcntl.h>
+
+#include "hyper.h"
+#include "util.h"
+#include "../config.h"
+
+int hyper_init_modules() 
+{
+	int status = hyper_cmd("/sbin/depmod");
+	if (status != 0) {
+		fprintf(stderr, "depmod failed, status: %d\n", status);
+		return -1;
+	}
+
+	return 0;
+}
+
+int hyper_setup_iptables_rule(struct ipt_rule rule)
+{
+	char check_cmd[512] = {0};
+	char cmd[512] = {0};
+	int check = -1;
+
+	if (rule.rule != NULL) {
+		sprintf(check_cmd, "/sbin/iptables -t %s -C %s %s", rule.table, rule.chain, rule.rule);
+		sprintf(cmd, "/sbin/iptables -t %s %s %s %s", rule.table, rule.op, rule.chain, rule.rule);
+	} else {
+		sprintf(cmd, "/sbin/iptables -t %s %s %s", rule.table, rule.op, rule.chain);
+	}
+
+	if (strlen(check_cmd) > 0) {
+		check = hyper_cmd(check_cmd);
+		fprintf(stdout, "check iptables '%s', ret: %d\n", check_cmd, check);
+	}
+
+	if (check == 0) {
+		// iptables rule already exist, do not insert it again
+		if (!strncmp(rule.op, "-A", strlen("-A")) ||
+			!strncmp(rule.op, "-I", strlen("-I")) ||
+			!strncmp(rule.op, "-N", strlen("-N"))) {
+			fprintf(stdout, "iptables rule '%s' already exist\n", rule.rule);
+			return 0;
+		}
+	}
+
+	int status = hyper_cmd(cmd);
+	fprintf(stdout, "insert iptables '%s', ret: %d\n", cmd, status);
+	if (status != 0) {
+		fprintf(stderr, "insert iptables rule failed, ret: %d\n", status);
+		return -1;
+	}
+
+	return 0;
+}
+
+// initialize modules and iptables chains
+int hyper_setup_portmapping(struct hyper_pod *pod)
+{
+	if (pod->w_num == 0) {
+		return 0;
+	}
+
+	if (hyper_init_modules() < 0) {
+		return -1;
+	}
+
+	// iptables -t filter -N hyperstart-INPUT
+	// iptables -t nat -N hyperstart-PREROUTING
+	// iptables -t filter -I INPUT -j hyperstart-INPUT
+	// iptables -t nat -I PREROUTING -j hyperstart-PREROUTING
+	// iptables -t filter -A hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+	// iptables -t filter -A hyperstart-INPUT -p icmp -j ACCEPT
+	// iptables -t filter -A hyperstart-INPUT -i lo -j ACCEPT
+	// iptables -t filter -A hyperstart-INPUT -j DROP
+	// iptables -t nat -A hyperstart-PREROUTING -j RETURN
+	const struct ipt_rule rules[] = {
+		{
+			.table = "filter",
+			.op = "-N",
+			.chain = "hyperstart-INPUT",
+			.rule = NULL,
+		},
+		{
+			.table = "nat",
+			.op = "-N",
+			.chain = "hyperstart-PREROUTING",
+			.rule = NULL,
+		},
+		{
+			.table = "filter",
+			.op = "-I",
+			.chain = "INPUT",
+			.rule = "-j hyperstart-INPUT",
+		},
+		{
+			.table = "nat",
+			.op = "-I",
+			.chain = "PREROUTING",
+			.rule = "-j hyperstart-PREROUTING",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-m state --state RELATED,ESTABLISHED -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-p icmp -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-i lo -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-A",
+			.chain = "hyperstart-INPUT",
+			.rule = "-j DROP",
+		},
+		{
+			.table = "nat",
+			.op = "-A",
+			.chain = "hyperstart-PREROUTING",
+			.rule = "-j RETURN",
+		},
+	};
+
+	int i = 0;
+	for(i=0; i< sizeof(rules)/sizeof(struct ipt_rule); i++) {
+		if (hyper_setup_iptables_rule(rules[i])<0) {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+void hyper_cleanup_portmapping(struct hyper_pod *pod)
+{
+	if (pod->w_num == 0) {
+		return;
+	}
+
+	// iptables -t filter -D hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+	// iptables -t filter -D hyperstart-INPUT -p icmp -j ACCEPT
+	// iptables -t filter -D hyperstart-INPUT -i lo -j ACCEPT
+	// iptables -t filter -D hyperstart-INPUT -j DROP
+	// iptables -t filter -D INPUT -j hyperstart-DNPUT
+	// iptables -t nat -D hyperstart-PREROUTING -j RETURN
+	// iptables -t nat -D PREROUTING -j hyperstart-PREROUTING
+	// iptables -t filter -F hyperstart-INPUT
+	// iptables -t nat -F hyperstart-PREROUTING
+	// iptables -t filter -X hyperstart-INPUT
+	// iptables -t nat -X hyperstart-PREROUTING
+	const struct ipt_rule rules[] = {
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-m state --state RELATED,ESTABLISHED -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-p icmp -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-i lo -j ACCEPT",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "hyperstart-INPUT",
+			.rule = "-j DROP",
+		},
+		{
+			.table = "nat",
+			.op = "-D",
+			.chain = "hyperstart-PREROUTING",
+			.rule = "-j RETURN",
+		},
+		{
+			.table = "nat",
+			.op = "-D",
+			.chain = "PREROUTING",
+			.rule = "-j hyperstart-PREROUTING",
+		},
+		{
+			.table = "filter",
+			.op = "-D",
+			.chain = "INPUT",
+			.rule = "-j hyperstart-INPUT",
+		},
+		{
+			.table = "nat",
+			.op = "-F",
+			.chain = "hyperstart-PREROUTING",
+			.rule = NULL,
+		},
+		{
+			.table = "nat",
+			.op = "-X",
+			.chain = "hyperstart-PREROUTING",
+			.rule = NULL,
+		},
+		{
+			.table = "filter",
+			.op = "-F",
+			.chain = "hyperstart-INPUT",
+			.rule = NULL,
+		},
+		{
+			.table = "filter",
+			.op = "-X",
+			.chain = "hyperstart-INPUT",
+			.rule = NULL,
+		},
+	};
+
+	int i = 0;
+	for(i=0; i< sizeof(rules)/sizeof(struct ipt_rule); i++) {
+		if (hyper_setup_iptables_rule(rules[i])<0) {
+			return;
+		}
+	}
+}
+
+int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod)
+{
+	if (pod->w_num == 0) {
+		return 0;
+	}
+
+	if (c->ports_num == 0) {
+		return 0;
+	}
+
+	int i = 0, j = 0;
+	char rule[128] = {0};
+
+	for (i=0; i<c->ports_num; i++) {
+		sprintf(rule, "-p %s -m %s --dport %d -j REDIRECT --to-ports %d",
+			c->ports[i].protocol,
+			c->ports[i].protocol,
+			c->ports[i].host_port,
+			c->ports[i].container_port);
+		struct ipt_rule rediect_rule = {
+			.table = "nat",
+			.op = "-I",
+			.chain = "hyperstart-PREROUTING",
+			.rule = rule,
+		};
+		if (hyper_setup_iptables_rule(rediect_rule)<0) {
+			fprintf(stderr, "setup rediect_rule '%s' failed\n", rule);
+			return -1;
+		}
+
+		for (j=0; j<pod->w_num; j++) {
+			sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
+				pod->white_cidrs[j],
+				c->ports[i].protocol,
+				c->ports[i].protocol,
+				c->ports[i].container_port);
+			struct ipt_rule accept_rule = {
+				.table = "filter",
+				.op = "-I",
+				.chain = "hyperstart-INPUT",
+				.rule = rule,
+			};
+			if (hyper_setup_iptables_rule(accept_rule)<0) {
+				fprintf(stderr, "setup accept_rule '%s' failed\n", rule);
+				return -1;
+			}
+		}
+		
+	}
+
+	return 0;
+}
+
+void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod)
+{
+	if (pod->w_num == 0) {
+		return;
+	}
+
+	if (c->ports_num == 0) {
+		return;
+	}
+
+
+	int i = 0, j = 0;
+	char rule[128] = {0};
+
+	for (i=0; i<c->ports_num; i++) {
+		sprintf(rule, "-p %s -m %s --dport %d -j REDIRECT --to-ports %d", 
+        c->ports[i].protocol,
+        c->ports[i].protocol,
+        c->ports[i].host_port,
+        c->ports[i].container_port);
+		struct ipt_rule rediect_rule = {
+			.table = "nat",
+			.op = "-D",
+			.chain = "hyperstart-PREROUTING",
+			.rule = rule,
+		};
+		if (hyper_setup_iptables_rule(rediect_rule)<0) {
+			fprintf(stderr, "setup rediect_rule '%s' failed\n", rule);
+		}
+
+		for (j=0; j<pod->w_num; j++) {
+			sprintf(rule, "-s %s -p %s -m %s --dport %d -j ACCEPT",
+				pod->white_cidrs[j],
+				c->ports[i].protocol,
+				c->ports[i].protocol,
+				c->ports[i].container_port);
+			struct ipt_rule accept_rule = {
+				.table = "filter",
+				.op = "-D",
+				.chain = "hyperstart-INPUT",
+				.rule = rule,
+			};
+			if (hyper_setup_iptables_rule(accept_rule)<0) {
+				fprintf(stderr, "setup accept_rule '%s' failed\n", rule);
+			}
+		}
+		
+	}
+}

src/portmapping.h

@@ -0,0 +1,23 @@
+#ifndef _PORT_MAPPING_H_
+#define _PORT_MAPPING_H_
+
+#include <stdio.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+struct ipt_rule {
+	char *table;
+	char *op;
+	char *chain;
+	char *rule;
+};
+
+struct hyper_pod;
+struct hyper_container;
+int hyper_setup_portmapping(struct hyper_pod *pod);
+void hyper_cleanup_portmapping(struct hyper_pod *pod);
+int hyper_setup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod);
+void hyper_cleanup_container_portmapping(struct hyper_container *c, struct hyper_pod *pod);
+
+#endif

src/util.c

@@ -625,3 +625,33 @@ void hyper_shutdown()
 	hyper_unmount_all();
 	reboot(LINUX_REBOOT_CMD_POWER_OFF);
 }
+
+int hyper_cmd(char *cmd)
+{
+	int pid, status;
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fail to fork");
+		return -1;
+	} else if (pid > 0) {
+		if (waitpid(pid, &status, 0) <= 0) {
+			perror("waiting fork cmd failed");
+			return -1;
+		}
+		if (WIFEXITED(status)) {
+			int ret = WEXITSTATUS(status);
+			fprintf(stdout, "%s cmd exit normally, status %" PRIu8 "\n", cmd, ret);
+			if (ret == 0)
+				return 0;
+		}
+
+		fprintf(stdout, "cmd %s exit unexpectedly, status %" PRIu8 "\n", cmd, status);
+		return -1;
+	} else {
+		fprintf(stdout, "executing cmd %s\n", cmd);
+		execlp("/busybox", "sh", "-c", cmd, NULL);
+	}
+
+	return -1;
+}

src/util.h

@@ -24,6 +24,7 @@ int hyper_copy_dir(char *src, char *dst);
 void hyper_sync_time_hctosys();
 void online_cpu(void);
 void online_memory(void);
+int hyper_cmd(char *cmd);
 int hyper_mkdir(char *path);
 int hyper_open_channel(char *channel, int mode);
 int hyper_open_serial_dev(char *tty);