From 356555864a78722cb66d3fc43b2d0e0a570b8618 Mon Sep 17 00:00:00 2001 From: Eugene Yakubovich Date: Fri, 6 Feb 2015 16:07:46 -0800 Subject: [PATCH] net: added IP masquerade support --- Godeps/Godeps.json | 4 + .../coreos/go-iptables/iptables/iptables.go | 255 ++++++++++++++++++ networking/net-plugin.go | 2 +- networking/net/bridge/bridge.go | 15 ++ networking/net/veth/veth.go | 81 ++++-- networking/util/ipmasq.go | 52 ++++ networking/util/link.go | 24 +- stage1/rootfs/net/99-default.conf | 3 +- 8 files changed, 409 insertions(+), 27 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/coreos/go-iptables/iptables/iptables.go create mode 100644 networking/util/ipmasq.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 70aabf1..1a999dc 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -35,6 +35,10 @@ "Comment": "v0.3.0-2-g4c3cbeae4798", "Rev": "4c3cbeae47980d073828dd263a8c982667855f06" }, + { + "ImportPath": "github.com/coreos/go-iptables/iptables", + "Rev": "83dfad0f13fd7310fb3c1cb8563248d8d604b95b" + }, { "ImportPath": "github.com/coreos/go-semver/semver", "Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa" diff --git a/Godeps/_workspace/src/github.com/coreos/go-iptables/iptables/iptables.go b/Godeps/_workspace/src/github.com/coreos/go-iptables/iptables/iptables.go new file mode 100644 index 0000000..36a8ec7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-iptables/iptables/iptables.go @@ -0,0 +1,255 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iptables + +import ( + "bytes" + "fmt" + "log" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" +) + +// Adds the output of stderr to exec.ExitError +type Error struct { + exec.ExitError + msg string +} + +func (e *Error) ExitStatus() int { + return e.Sys().(syscall.WaitStatus).ExitStatus() +} + +func (e *Error) Error() string { + return fmt.Sprintf("exit status %v: %v", e.ExitStatus(), e.msg) +} + +type IPTables struct { + path string +} + +func New() (*IPTables, error) { + path, err := exec.LookPath("iptables") + if err != nil { + return nil, err + } + + return &IPTables{path}, nil +} + +// Exists checks if given rulespec in specified table/chain exists +func (ipt *IPTables) Exists(table, chain string, rulespec...string) (bool, error) { + checkPresent, err := getIptablesHasCheckCommand() + if err != nil { + log.Printf("Error checking iptables version, assuming version at least 1.4.11: %v", err) + checkPresent = true + } + + if !checkPresent { + cmd := append([]string{"-A", chain}, rulespec...) + return existsForOldIpTables(table, strings.Join(cmd, " ")) + } else { + cmd := append([]string{"-t", table, "-C", chain}, rulespec...) + err := ipt.run(cmd...) + + switch { + case err == nil: + return true, nil + case err.(*Error).ExitStatus() == 1: + return false, nil + default: + return false, err + } + } +} + +// Insert inserts rulespec to specified table/chain (in specified pos) +func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error { + cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...) + return ipt.run(cmd...) +} + +// Append appends rulespec to specified table/chain +func (ipt *IPTables) Append(table, chain string, rulespec ...string) error { + cmd := append([]string{"-t", table, "-A", chain}, rulespec...) + return ipt.run(cmd...) +} + +// AppendUnique acts like Append except that it won't add a duplicate +func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error { + exists, err := ipt.Exists(table, chain, rulespec...) + if err != nil { + return err + } + + if !exists { + return ipt.Append(table, chain, rulespec...) + } + + return nil +} + +// Delete removes rulespec in specified table/chain +func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error { + cmd := append([]string{"-t", table, "-D", chain}, rulespec...) + return ipt.run(cmd...) +} + +// List rules in specified table/chain +func (ipt *IPTables) List(table, chain string) ([]string, error) { + var stdout, stderr bytes.Buffer + cmd := exec.Cmd{ + Path: ipt.path, + Args: []string{ipt.path, "-t", table, "-S", chain}, + Stdout: &stdout, + Stderr: &stderr, + } + + if err := cmd.Run(); err != nil { + return nil, &Error{*(err.(*exec.ExitError)), stderr.String()} + } + + rules := strings.Split(stdout.String(), "\n") + if len(rules) > 0 && rules[len(rules)-1] == "" { + rules = rules[:len(rules)-1] + } + + return rules, nil +} + +func (ipt *IPTables) NewChain(table, chain string) error { + return ipt.run("-t", table, "-N", chain) +} + +// ClearChain flushed (deletes all rules) in the specifed table/chain. +// If the chain does not exist, new one will be created +func (ipt *IPTables) ClearChain(table, chain string) error { + err := ipt.NewChain(table, chain) + + switch { + case err == nil: + return nil + case err.(*Error).ExitStatus() == 1: + // chain already exists. Flush (clear) it. + return ipt.run("-t", table, "-F", chain) + default: + return err + } +} + +// DeleteChain deletes the chain in the specified table. +// The chain must be empty +func (ipt *IPTables) DeleteChain(table, chain string) error { + return ipt.run("-t", table, "-X", chain) +} + +func (ipt *IPTables) run(args... string) error { + var stderr bytes.Buffer + cmd := exec.Cmd{ + Path: ipt.path, + Args: append([]string{ipt.path}, args...), + Stderr: &stderr, + } + + if err := cmd.Run(); err != nil { + return &Error{*(err.(*exec.ExitError)), stderr.String()} + } + + return nil +} + +// Checks if iptables has the "-C" flag +func getIptablesHasCheckCommand() (bool, error) { + vstring, err := getIptablesVersionString() + if err != nil { + return false, err + } + + v1, v2, v3, err := extractIptablesVersion(vstring) + if err != nil { + return false, err + } + + return iptablesHasCheckCommand(v1, v2, v3), nil +} + +// getIptablesVersion returns the first three components of the iptables version. +// e.g. "iptables v1.3.66" would return (1, 3, 66, nil) +func extractIptablesVersion(str string) (int, int, int, error) { + versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)") + result := versionMatcher.FindStringSubmatch(str) + if result == nil { + return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str) + } + + v1, err := strconv.Atoi(result[1]) + if err != nil { + return 0, 0, 0, err + } + + v2, err := strconv.Atoi(result[2]) + if err != nil { + return 0, 0, 0, err + } + + v3, err := strconv.Atoi(result[3]) + if err != nil { + return 0, 0, 0, err + } + + return v1, v2, v3, nil +} + +// Runs "iptables --version" to get the version string +func getIptablesVersionString() (string, error) { + cmd := exec.Command("iptables", "--version") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return "", err + } + return out.String(), nil +} + +// Checks if an iptables version is after 1.4.11, when --check was added +func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool { + if v1 > 1 { + return true + } + if v1 == 1 && v2 > 4 { + return true + } + if v1 == 1 && v2 == 4 && v3 >= 11 { + return true + } + return false +} + +// Checks if a rule specification exists for a table +func existsForOldIpTables(table string, ruleSpec string) (bool, error) { + cmd := exec.Command("iptables", "-t", table, "-S") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return false, err + } + rules := out.String() + return strings.Contains(rules, ruleSpec), nil +} diff --git a/networking/net-plugin.go b/networking/net-plugin.go index e87e5cf..d673f42 100644 --- a/networking/net-plugin.go +++ b/networking/net-plugin.go @@ -71,7 +71,7 @@ func (e *containerEnv) findNetPlugin(plugin string) string { } func envVars(vars [][2]string) []string { - env := []string{} + env := os.Environ() for _, kv := range vars { env = append(env, strings.Join(kv[:], "=")) diff --git a/networking/net/bridge/bridge.go b/networking/net/bridge/bridge.go index 0616808..2dd6a8e 100644 --- a/networking/net/bridge/bridge.go +++ b/networking/net/bridge/bridge.go @@ -36,6 +36,7 @@ type Net struct { rktnet.Net BrName string `json:"bridgeName"` IsGW bool `json:"isGateway"` + IPMasq bool `json:"ipMasq"` } func init() { @@ -210,11 +211,25 @@ func cmdAdd(contID, netns, netConf, ifName string) error { return err } + if n.IPMasq { + chain := "RKT-" + n.Name + if err = util.SetupIPMasq(network(ipConf.IP), chain); err != nil { + return err + } + } + return rktnet.PrintIfConfig(&rktnet.IfConfig{ IP: ipConf.IP.IP, }) } +func network(ipn *net.IPNet) *net.IPNet { + return &net.IPNet{ + IP: ipn.IP.Mask(ipn.Mask), + Mask: ipn.Mask, + } +} + func cmdDel(contID, netns, netConf, ifName string) error { n, err := loadConf(netConf) if err != nil { diff --git a/networking/net/veth/veth.go b/networking/net/veth/veth.go index 66e311c..6e4218d 100644 --- a/networking/net/veth/veth.go +++ b/networking/net/veth/veth.go @@ -35,21 +35,14 @@ func init() { runtime.LockOSThread() } -func cmdAdd(contID, netns, netConf, ifName, args string) error { +type Net struct { + rktnet.Net + IPMasq bool `json:"ipMasq"` +} + +func setupContVeth(contID, netns, ifName string, ipConf *ipam.IPConfig) (string, error) { var hostVethName string - - conf := rktnet.Net{} - if err := rktnet.LoadNet(netConf, &conf); err != nil { - return fmt.Errorf("failed to load %q: %v", netConf, err) - } - - // run the IPAM plugin and get back the config to apply - ipConf, err := ipam.ExecPluginAdd(conf.IPAM.Type) - if err != nil { - return err - } - - err = util.WithNetNSPath(netns, func(hostNS *os.File) error { + err := util.WithNetNSPath(netns, func(hostNS *os.File) error { entropy := contID + ifName hostVeth, _, err := util.SetupVeth(entropy, ifName, nil, hostNS) @@ -66,48 +59,90 @@ func cmdAdd(contID, netns, netConf, ifName, args string) error { return nil }) - if err != nil { - return err - } + return hostVethName, err +} +func setupHostVeth(vethName string, ipConf *ipam.IPConfig) error { // hostVeth moved namespaces and may have a new ifindex - hostVeth, err := netlink.LinkByName(hostVethName) + veth, err := netlink.LinkByName(vethName) if err != nil { - return fmt.Errorf("failed to lookup %q: %v", hostVethName, err) + return fmt.Errorf("failed to lookup %q: %v", vethName, err) } + // TODO(eyakubovich): IPv6 ipn := &net.IPNet{ IP: ipConf.Gateway, Mask: net.CIDRMask(31, 32), } addr := &netlink.Addr{IPNet: ipn, Label: ""} - if err = netlink.AddrAdd(hostVeth, addr); err != nil { + if err = netlink.AddrAdd(veth, addr); err != nil { return fmt.Errorf("failed to add IP addr to veth: %v", err) } // dst happens to be the same as IP/net of host veth - if err = util.AddHostRoute(ipn, nil, hostVeth); err != nil && !os.IsExist(err) { + if err = util.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) { return fmt.Errorf("failed to add route on host: %v", err) } + return nil +} + +func cmdAdd(contID, netns, netConf, ifName, args string) error { + conf := Net{} + if err := rktnet.LoadNet(netConf, &conf); err != nil { + return fmt.Errorf("failed to load %q: %v", netConf, err) + } + + // run the IPAM plugin and get back the config to apply + ipConf, err := ipam.ExecPluginAdd(conf.IPAM.Type) + if err != nil { + return err + } + + hostVethName, err := setupContVeth(contID, netns, ifName, ipConf) + if err != nil { + return err + } + + if err = setupHostVeth(hostVethName, ipConf); err != nil { + return err + } + + if conf.IPMasq { + chain := fmt.Sprintf("RKT-%s-%s", conf.Name, contID[:8]) + if err = util.SetupIPMasq(ipConf.IP, chain); err != nil { + return err + } + } + return rktnet.PrintIfConfig(&rktnet.IfConfig{ IP: ipConf.IP.IP, }) } func cmdDel(contID, netns, netConf, ifName, args string) error { - conf := rktnet.Net{} + conf := Net{} if err := rktnet.LoadNet(netConf, &conf); err != nil { return fmt.Errorf("failed to load %q: %v", netConf, err) } + var ipn *net.IPNet err := util.WithNetNSPath(netns, func(hostNS *os.File) error { - return util.DelLinkByName(ifName) + var err error + ipn, err = util.DelLinkByNameAddr(ifName, netlink.FAMILY_V4) + return err }) if err != nil { return err } + if conf.IPMasq { + chain := fmt.Sprintf("RKT-%s-%s", conf.Name, contID[:8]) + if err = util.TeardownIPMasq(ipn, chain); err != nil { + return err + } + } + return ipam.ExecPluginDel(conf.IPAM.Type) } diff --git a/networking/util/ipmasq.go b/networking/util/ipmasq.go new file mode 100644 index 0000000..7a10936 --- /dev/null +++ b/networking/util/ipmasq.go @@ -0,0 +1,52 @@ +package util + +import ( + "fmt" + "net" + + "github.com/coreos/rocket/Godeps/_workspace/src/github.com/coreos/go-iptables/iptables" +) + +// Installs iptables rules to masquerade traffic +// coming from ipn and going outside of it +func SetupIPMasq(ipn *net.IPNet, chain string) error { + ipt, err := iptables.New() + if err != nil { + return fmt.Errorf("failed to locate iptabes: %v", err) + } + + if err = ipt.NewChain("nat", chain); err != nil { + if err.(*iptables.Error).ExitStatus() != 1 { + // TODO(eyakubovich): assumes exit status 1 implies chain exists + return err + } + } + + if err = ipt.AppendUnique("nat", chain, "-d", ipn.String(), "-j", "ACCEPT"); err != nil { + return err + } + + if err = ipt.AppendUnique("nat", chain, "!", "-d", "224.0.0.0/4", "-j", "MASQUERADE"); err != nil { + return err + } + + return ipt.AppendUnique("nat", "POSTROUTING", "-s", ipn.String(), "-j", chain) +} + +// Undoes the effects of SetupIPMasq +func TeardownIPMasq(ipn *net.IPNet, chain string) error { + ipt, err := iptables.New() + if err != nil { + return fmt.Errorf("failed to locate iptabes: %v", err) + } + + if err = ipt.Delete("nat", "POSTROUTING", "-s", ipn.String(), "-j", chain); err != nil { + return err + } + + if err = ipt.ClearChain("nat", chain); err != nil { + return err + } + + return ipt.DeleteChain("nat", chain) +} diff --git a/networking/util/link.go b/networking/util/link.go index 014c3b9..72877f8 100644 --- a/networking/util/link.go +++ b/networking/util/link.go @@ -92,12 +92,32 @@ func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (h func DelLinkByName(ifName string) error { iface, err := netlink.LinkByName(ifName) if err != nil { - return fmt.Errorf("Failed to lookup %q: %v", ifName, err) + return fmt.Errorf("failed to lookup %q: %v", ifName, err) } if err = netlink.LinkDel(iface); err != nil { - return fmt.Errorf("Failed to delete %q: %v", ifName, err) + return fmt.Errorf("failed to delete %q: %v", ifName, err) } return nil } + +// DelLinkByNameAddr remove an interface returns its IP address +// of the specified family +func DelLinkByNameAddr(ifName string, family int) (*net.IPNet, error) { + iface, err := netlink.LinkByName(ifName) + if err != nil { + return nil, fmt.Errorf("failed to lookup %q: %v", ifName, err) + } + + addrs, err := netlink.AddrList(iface, family) + if err != nil || len(addrs) == 0 { + return nil, fmt.Errorf("failed to get IP addresses for %q: %v", ifName, err) + } + + if err = netlink.LinkDel(iface); err != nil { + return nil, fmt.Errorf("failed to delete %q: %v", ifName, err) + } + + return addrs[0].IPNet, nil +} diff --git a/stage1/rootfs/net/99-default.conf b/stage1/rootfs/net/99-default.conf index 481a139..f2e907a 100644 --- a/stage1/rootfs/net/99-default.conf +++ b/stage1/rootfs/net/99-default.conf @@ -5,5 +5,6 @@ "type": "static-ptp", "subnet": "172.16.28.0/24", "routes": [ "169.254.169.255/32", "0.0.0.0/0" ] - } + }, + "ipMasq": true }