Merge pull request #498 from eyakubovich/ipmasq

net: added IP masquerade support

Godeps/Godeps.json

@@ -35,6 +35,10 @@
 			"Comment": "v0.3.0-2-g4c3cbeae4798",
 			"Rev": "4c3cbeae47980d073828dd263a8c982667855f06"
 		},
+		{
+			"ImportPath": "github.com/coreos/go-iptables/iptables",
+			"Rev": "83dfad0f13fd7310fb3c1cb8563248d8d604b95b"
+		},
 		{
 			"ImportPath": "github.com/coreos/go-semver/semver",
 			"Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa"

Godeps/_workspace/src/github.com/coreos/go-iptables/iptables/iptables.go

@@ -0,0 +1,255 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+// Adds the output of stderr to exec.ExitError
+type Error struct {
+	exec.ExitError
+	msg string
+}
+
+func (e *Error) ExitStatus() int {
+	return e.Sys().(syscall.WaitStatus).ExitStatus()
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("exit status %v: %v", e.ExitStatus(), e.msg)
+}
+
+type IPTables struct {
+	path string
+}
+
+func New() (*IPTables, error) {
+	path, err := exec.LookPath("iptables")
+	if err != nil {
+		return nil, err
+	}
+
+	return &IPTables{path}, nil
+}
+
+// Exists checks if given rulespec in specified table/chain exists
+func (ipt *IPTables) Exists(table, chain string, rulespec...string) (bool, error) {
+	checkPresent, err := getIptablesHasCheckCommand()
+	if err != nil {
+		log.Printf("Error checking iptables version, assuming version at least 1.4.11: %v", err)
+		checkPresent = true
+	}
+
+	if !checkPresent {
+		cmd := append([]string{"-A", chain}, rulespec...)
+		return existsForOldIpTables(table, strings.Join(cmd, " "))
+	} else {
+		cmd := append([]string{"-t", table, "-C", chain}, rulespec...)
+		err := ipt.run(cmd...)
+
+		switch {
+		case err == nil:
+			return true, nil
+		case err.(*Error).ExitStatus() == 1:
+			return false, nil
+		default:
+			return false, err
+		}
+	}
+}
+
+// Insert inserts rulespec to specified table/chain (in specified pos)
+func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// Append appends rulespec to specified table/chain
+func (ipt *IPTables) Append(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-A", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// AppendUnique acts like Append except that it won't add a duplicate
+func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Append(table, chain, rulespec...)
+	}
+
+	return nil
+}
+
+// Delete removes rulespec in specified table/chain
+func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-D", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// List rules in specified table/chain
+func (ipt *IPTables) List(table, chain string) ([]string, error) {
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path: ipt.path,
+		Args: []string{ipt.path, "-t", table, "-S", chain},
+		Stdout: &stdout,
+		Stderr: &stderr,
+	}
+
+	if err := cmd.Run(); err != nil {
+		return nil, &Error{*(err.(*exec.ExitError)), stderr.String()}
+	}
+
+	rules := strings.Split(stdout.String(), "\n")
+	if len(rules) > 0 && rules[len(rules)-1] == "" {
+		rules = rules[:len(rules)-1]
+	}
+
+	return rules, nil
+}
+
+func (ipt *IPTables) NewChain(table, chain string) error {
+	return ipt.run("-t", table, "-N", chain)
+}
+
+// ClearChain flushed (deletes all rules) in the specifed table/chain.
+// If the chain does not exist, new one will be created
+func (ipt *IPTables) ClearChain(table, chain string) error {
+	err := ipt.NewChain(table, chain)
+
+	switch {
+	case err == nil:
+		return nil
+	case err.(*Error).ExitStatus() == 1:
+		// chain already exists. Flush (clear) it.
+		return ipt.run("-t", table, "-F", chain)
+	default:
+		return err
+	}
+}
+
+// DeleteChain deletes the chain in the specified table.
+// The chain must be empty
+func (ipt *IPTables) DeleteChain(table, chain string) error {
+	return ipt.run("-t", table, "-X", chain)
+}
+
+func (ipt *IPTables) run(args... string) error {
+	var stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path: ipt.path,
+		Args: append([]string{ipt.path}, args...),
+		Stderr: &stderr,
+	}
+
+	if err := cmd.Run(); err != nil {
+		return &Error{*(err.(*exec.ExitError)), stderr.String()}
+	}
+
+	return nil
+}
+
+// Checks if iptables has the "-C" flag
+func getIptablesHasCheckCommand() (bool, error) {
+	vstring, err := getIptablesVersionString()
+	if err != nil {
+		return false, err
+	}
+
+	v1, v2, v3, err := extractIptablesVersion(vstring)
+	if err != nil {
+		return false, err
+	}
+
+	return iptablesHasCheckCommand(v1, v2, v3), nil
+}
+
+// getIptablesVersion returns the first three components of the iptables version.
+// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
+func extractIptablesVersion(str string) (int, int, int, error) {
+	versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
+	result := versionMatcher.FindStringSubmatch(str)
+	if result == nil {
+		return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
+	}
+
+	v1, err := strconv.Atoi(result[1])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	v2, err := strconv.Atoi(result[2])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	v3, err := strconv.Atoi(result[3])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	return v1, v2, v3, nil
+}
+
+// Runs "iptables --version" to get the version string
+func getIptablesVersionString() (string, error) {
+	cmd := exec.Command("iptables", "--version")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+// Checks if an iptables version is after 1.4.11, when --check was added
+func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 11 {
+		return true
+	}
+	return false
+}
+
+// Checks if a rule specification exists for a table
+func existsForOldIpTables(table string, ruleSpec string) (bool, error) {
+	cmd := exec.Command("iptables", "-t", table, "-S")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return false, err
+	}
+	rules := out.String()
+	return strings.Contains(rules, ruleSpec), nil
+}

networking/net-plugin.go

@@ -71,7 +71,7 @@ func (e *containerEnv) findNetPlugin(plugin string) string {
 }
 
 func envVars(vars [][2]string) []string {
-	env := []string{}
+	env := os.Environ()
 
 	for _, kv := range vars {
 		env = append(env, strings.Join(kv[:], "="))

networking/net/bridge/bridge.go

@@ -36,6 +36,7 @@ type Net struct {
 	rktnet.Net
 	BrName string `json:"bridgeName"`
 	IsGW   bool   `json:"isGateway"`
+	IPMasq bool   `json:"ipMasq"`
 }
 
 func init() {
@@ -210,11 +211,25 @@ func cmdAdd(contID, netns, netConf, ifName string) error {
 		return err
 	}
 
+	if n.IPMasq {
+		chain := "RKT-" + n.Name
+		if err = util.SetupIPMasq(network(ipConf.IP), chain); err != nil {
+			return err
+		}
+	}
+
 	return rktnet.PrintIfConfig(&rktnet.IfConfig{
 		IP: ipConf.IP.IP,
 	})
 }
 
+func network(ipn *net.IPNet) *net.IPNet {
+	return &net.IPNet{
+		IP:   ipn.IP.Mask(ipn.Mask),
+		Mask: ipn.Mask,
+	}
+}
+
 func cmdDel(contID, netns, netConf, ifName string) error {
 	n, err := loadConf(netConf)
 	if err != nil {

networking/net/veth/veth.go

@@ -35,21 +35,14 @@ func init() {
 	runtime.LockOSThread()
 }
 
-func cmdAdd(contID, netns, netConf, ifName, args string) error {
+type Net struct {
+	rktnet.Net
+	IPMasq bool `json:"ipMasq"`
+}
+
+func setupContVeth(contID, netns, ifName string, ipConf *ipam.IPConfig) (string, error) {
 	var hostVethName string
-
-	conf := rktnet.Net{}
-	if err := rktnet.LoadNet(netConf, &conf); err != nil {
-		return fmt.Errorf("failed to load %q: %v", netConf, err)
-	}
-
-	// run the IPAM plugin and get back the config to apply
-	ipConf, err := ipam.ExecPluginAdd(conf.IPAM.Type)
-	if err != nil {
-		return err
-	}
-
-	err = util.WithNetNSPath(netns, func(hostNS *os.File) error {
+	err := util.WithNetNSPath(netns, func(hostNS *os.File) error {
 		entropy := contID + ifName
 
 		hostVeth, _, err := util.SetupVeth(entropy, ifName, nil, hostNS)
@@ -66,48 +59,90 @@ func cmdAdd(contID, netns, netConf, ifName, args string) error {
 
 		return nil
 	})
-	if err != nil {
-		return err
-	}
+	return hostVethName, err
+}
 
+func setupHostVeth(vethName string, ipConf *ipam.IPConfig) error {
 	// hostVeth moved namespaces and may have a new ifindex
-	hostVeth, err := netlink.LinkByName(hostVethName)
+	veth, err := netlink.LinkByName(vethName)
 	if err != nil {
-		return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
+		return fmt.Errorf("failed to lookup %q: %v", vethName, err)
 	}
 
+	// TODO(eyakubovich): IPv6
 	ipn := &net.IPNet{
 		IP:   ipConf.Gateway,
 		Mask: net.CIDRMask(31, 32),
 	}
 	addr := &netlink.Addr{IPNet: ipn, Label: ""}
-	if err = netlink.AddrAdd(hostVeth, addr); err != nil {
+	if err = netlink.AddrAdd(veth, addr); err != nil {
 		return fmt.Errorf("failed to add IP addr to veth: %v", err)
 	}
 
 	// dst happens to be the same as IP/net of host veth
-	if err = util.AddHostRoute(ipn, nil, hostVeth); err != nil && !os.IsExist(err) {
+	if err = util.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) {
 		return fmt.Errorf("failed to add route on host: %v", err)
 	}
 
+	return nil
+}
+
+func cmdAdd(contID, netns, netConf, ifName, args string) error {
+	conf := Net{}
+	if err := rktnet.LoadNet(netConf, &conf); err != nil {
+		return fmt.Errorf("failed to load %q: %v", netConf, err)
+	}
+
+	// run the IPAM plugin and get back the config to apply
+	ipConf, err := ipam.ExecPluginAdd(conf.IPAM.Type)
+	if err != nil {
+		return err
+	}
+
+	hostVethName, err := setupContVeth(contID, netns, ifName, ipConf)
+	if err != nil {
+		return err
+	}
+
+	if err = setupHostVeth(hostVethName, ipConf); err != nil {
+		return err
+	}
+
+	if conf.IPMasq {
+		chain := fmt.Sprintf("RKT-%s-%s", conf.Name, contID[:8])
+		if err = util.SetupIPMasq(ipConf.IP, chain); err != nil {
+			return err
+		}
+	}
+
 	return rktnet.PrintIfConfig(&rktnet.IfConfig{
 		IP: ipConf.IP.IP,
 	})
 }
 
 func cmdDel(contID, netns, netConf, ifName, args string) error {
-	conf := rktnet.Net{}
+	conf := Net{}
 	if err := rktnet.LoadNet(netConf, &conf); err != nil {
 		return fmt.Errorf("failed to load %q: %v", netConf, err)
 	}
 
+	var ipn *net.IPNet
 	err := util.WithNetNSPath(netns, func(hostNS *os.File) error {
-		return util.DelLinkByName(ifName)
+		var err error
+		ipn, err = util.DelLinkByNameAddr(ifName, netlink.FAMILY_V4)
+		return err
 	})
 	if err != nil {
 		return err
 	}
 
+	if conf.IPMasq {
+		chain := fmt.Sprintf("RKT-%s-%s", conf.Name, contID[:8])
+		if err = util.TeardownIPMasq(ipn, chain); err != nil {
+			return err
+		}
+	}
+
 	return ipam.ExecPluginDel(conf.IPAM.Type)
 }
 

networking/util/ipmasq.go

@@ -0,0 +1,52 @@
+package util
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/coreos/go-iptables/iptables"
+)
+
+// Installs iptables rules to masquerade traffic
+// coming from ipn and going outside of it
+func SetupIPMasq(ipn *net.IPNet, chain string) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return fmt.Errorf("failed to locate iptabes: %v", err)
+	}
+
+	if err = ipt.NewChain("nat", chain); err != nil {
+		if err.(*iptables.Error).ExitStatus() != 1 {
+			// TODO(eyakubovich): assumes exit status 1 implies chain exists
+			return err
+		}
+	}
+
+	if err = ipt.AppendUnique("nat", chain, "-d", ipn.String(), "-j", "ACCEPT"); err != nil {
+		return err
+	}
+
+	if err = ipt.AppendUnique("nat", chain, "!", "-d", "224.0.0.0/4", "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+
+	return ipt.AppendUnique("nat", "POSTROUTING", "-s", ipn.String(), "-j", chain)
+}
+
+// Undoes the effects of SetupIPMasq
+func TeardownIPMasq(ipn *net.IPNet, chain string) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return fmt.Errorf("failed to locate iptabes: %v", err)
+	}
+
+	if err = ipt.Delete("nat", "POSTROUTING", "-s", ipn.String(), "-j", chain); err != nil {
+		return err
+	}
+
+	if err = ipt.ClearChain("nat", chain); err != nil {
+		return err
+	}
+
+	return ipt.DeleteChain("nat", chain)
+}

networking/util/link.go

@@ -92,12 +92,32 @@ func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (h
 func DelLinkByName(ifName string) error {
 	iface, err := netlink.LinkByName(ifName)
 	if err != nil {
-		return fmt.Errorf("Failed to lookup %q: %v", ifName, err)
+		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
 	}
 
 	if err = netlink.LinkDel(iface); err != nil {
-		return fmt.Errorf("Failed to delete %q: %v", ifName, err)
+		return fmt.Errorf("failed to delete %q: %v", ifName, err)
 	}
 
 	return nil
 }
+
+// DelLinkByNameAddr remove an interface returns its IP address
+// of the specified family
+func DelLinkByNameAddr(ifName string, family int) (*net.IPNet, error) {
+	iface, err := netlink.LinkByName(ifName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to lookup %q: %v", ifName, err)
+	}
+
+	addrs, err := netlink.AddrList(iface, family)
+	if err != nil || len(addrs) == 0 {
+		return nil, fmt.Errorf("failed to get IP addresses for %q: %v", ifName, err)
+	}
+
+	if err = netlink.LinkDel(iface); err != nil {
+		return nil, fmt.Errorf("failed to delete %q: %v", ifName, err)
+	}
+
+	return addrs[0].IPNet, nil
+}

stage1/rootfs/net/99-default.conf

@@ -5,5 +5,6 @@
 		"type": "static-ptp",
 		"subnet": "172.16.28.0/24",
 		"routes": [ "169.254.169.255/32", "0.0.0.0/0" ]
-	}
+	},
+	"ipMasq": true
 }