From 696faf67b14481cbebec92a00742e3a544ebaa38 Mon Sep 17 00:00:00 2001 From: Eugene Yakubovich Date: Wed, 21 Jan 2015 12:48:17 -0800 Subject: [PATCH] net: Give the host end of veth an IP The scheme of having a host veth not have an IP and not be plugged into bridge does not work in practice. This creates a true point-to-point link between the container and the host. --- networking/ipam/ipam.go | 43 +++++++++++++++++++++---------- networking/networking.go | 5 ++-- networking/plugins/veth/veth.go | 45 +++++++++++++++++++++++---------- networking/util/link.go | 15 ++++++++--- networking/util/net.go | 8 +++--- networking/util/route.go | 8 ++++++ networking/util/setns.go | 8 +++--- 7 files changed, 90 insertions(+), 42 deletions(-) diff --git a/networking/ipam/ipam.go b/networking/ipam/ipam.go index 311cfc0..843c7ba 100644 --- a/networking/ipam/ipam.go +++ b/networking/ipam/ipam.go @@ -44,7 +44,7 @@ func ipAdd(ip net.IP, val uint) net.IP { return net.IP(nip) } -func allocIP(ipn *net.IPNet) (*net.IPNet, error) { +func allocIP(ipn *net.IPNet) (net.IP, error) { ones, bits := ipn.Mask.Size() zeros := bits - ones rng := (1 << uint(zeros)) - 2 // (reduce for gw, bcast) @@ -55,11 +55,7 @@ func allocIP(ipn *net.IPNet) (*net.IPNet, error) { } offset := uint(n.Uint64() + 1) - - return &net.IPNet{ - IP: ipAdd(ipn.IP, offset), - Mask: ipn.Mask, - }, nil + return ipAdd(ipn.IP, offset), nil } func deallocIP(ip net.IP) error { @@ -112,11 +108,15 @@ func AllocIP(contID types.UUID, netConf, ifName, args string) (*net.IPNet, net.I } if opts.ipRange != nil { - ipn, err := allocIP(opts.ipRange) + ip, err := allocIP(opts.ipRange) if err != nil { - return nil, nil, fmt.Errorf("error allocating IP in %v: %v", ipn, err) + return nil, nil, fmt.Errorf("error allocating IP in %v: %v", opts.ipRange, err) } - return ipn, nil, nil + + return &net.IPNet{ + IP: ip, + Mask: opts.ipRange.Mask, + }, nil, nil } n := util.Net{} @@ -126,25 +126,42 @@ func AllocIP(contID types.UUID, netConf, ifName, args string) (*net.IPNet, net.I switch n.IPAlloc.Type { case "static": - _, ipn, err := net.ParseCIDR(n.IPAlloc.Subnet) + _, rng, err := net.ParseCIDR(n.IPAlloc.Subnet) if err != nil { // TODO: cleanup return nil, nil, fmt.Errorf("error parsing %q conf: ipAlloc.Subnet: %v") } - ipn, err = allocIP(ipn) + ip, err := allocIP(rng) if err != nil { // TODO: cleanup - return nil, nil, fmt.Errorf("error allocating IP in %v: %v", ipn, err) + return nil, nil, fmt.Errorf("error allocating IP in %v: %v", rng, err) } - return ipn, nil, nil + return &net.IPNet{ + IP: ip, + Mask: rng.Mask, + }, nil, nil default: return nil, nil, fmt.Errorf("unsupported IP allocation type") } } +// allocates a /31 for point-to-point links +func AllocPtP(contID types.UUID, netConf, ifName, args string) ([2]net.IP, error) { + ipn, _, err := AllocIP(contID, netConf, ifName, args) + if err != nil { + return [2]net.IP{nil, nil}, err + } + + mask := net.CIDRMask(31, 32) + first := ipn.IP.Mask(mask) + second := ipAdd(first, 1) + + return [2]net.IP{first, second}, nil +} + func DeallocIP(contID types.UUID, netConf, ifName string, ipn *net.IPNet) error { return nil } diff --git a/networking/networking.go b/networking/networking.go index d95f1c4..d1547de 100644 --- a/networking/networking.go +++ b/networking/networking.go @@ -30,7 +30,7 @@ import ( const ( ifnamePattern = "eth%d" - selfNetNS = "/proc/self/ns/net" + selfNetNS = "/proc/self/ns/net" ) type activeNet struct { @@ -141,7 +141,6 @@ func basicNetNS() (hostNS, contNS *os.File, err error) { return } - func (n *Networking) EnterHostNS() error { return util.SetNS(n.hostNS, syscall.CLONE_NEWNET) } @@ -163,7 +162,7 @@ func setupNets(contID types.UUID, netns string, plugins map[string]*NetPlugin, n } an := activeNet{ - Net: nt, + Net: nt, ifName: fmt.Sprintf(ifnamePattern, i), } diff --git a/networking/plugins/veth/veth.go b/networking/plugins/veth/veth.go index 43796b7..c10fab9 100644 --- a/networking/plugins/veth/veth.go +++ b/networking/plugins/veth/veth.go @@ -36,7 +36,7 @@ func init() { } func cmdAdd(contID, netns, netConf, ifName, args string) error { - var hostVethName string + var hostVethName, contIPNet string cid, err := types.NewUUID(contID) if err != nil { @@ -48,14 +48,21 @@ func cmdAdd(contID, netns, netConf, ifName, args string) error { return fmt.Errorf("failed to load %q: %v", netConf, err) } - ipn, gw, err := ipam.AllocIP(*cid, netConf, ifName, args) + ips, err := ipam.AllocPtP(*cid, netConf, ifName, args) if err != nil { return err } + hostIP, contIP := ips[0], ips[1] + err = util.WithNetNSPath(netns, func(hostNS *os.File) error { entropy := contID + ifName + ipn := &net.IPNet{ + IP: contIP, + Mask: net.CIDRMask(31, 32), + } + hostVeth, contVeth, err := util.SetupVeth(entropy, ifName, ipn, hostNS) if err != nil { return err @@ -64,32 +71,44 @@ func cmdAdd(contID, netns, netConf, ifName, args string) error { for _, r := range conf.Routes { dst, err := util.ParseCIDR(r) if err != nil { - return err + return fmt.Errorf("failed to parse route %q: %v", r, err) } - if err = util.AddRoute(dst, gw, contVeth); err != nil { + if err = util.AddRoute(dst, hostIP, contVeth); err != nil { return fmt.Errorf("failed to add route %q: %v", dst, err) } } hostVethName = hostVeth.Attrs().Name - return err - }) + contIPNet = ipn.String() - // hostVeth moved namespaces and will have a new ifindex - hostVeth, err := netlink.LinkByName(hostVethName) + return nil + }) if err != nil { - return fmt.Errorf("failed to lookup %q: %v", hostVeth.Attrs().Name, err) + return err } - // On the host we route traffic for the allocated IP to the container - ipn.Mask = net.CIDRMask(32, 32) + // hostVeth moved namespaces and may have a new ifindex + hostVeth, err := netlink.LinkByName(hostVethName) + if err != nil { + return fmt.Errorf("failed to lookup %q: %v", hostVethName, err) + } - if err = util.AddRoute(ipn, nil, hostVeth); err != nil { + ipn := &net.IPNet{ + IP: hostIP, + Mask: net.CIDRMask(31, 32), + } + addr := &netlink.Addr{ipn, ""} + if err = netlink.AddrAdd(hostVeth, addr); err != nil { + return fmt.Errorf("failed to add IP addr to veth: %v", err) + } + + // dst happens to be the same as IP/net of host veth + if err = util.AddHostRoute(ipn, nil, hostVeth); err != nil && !os.IsExist(err) { return fmt.Errorf("failed to add route on host: %v", err) } - fmt.Print(ipn.String()) + fmt.Print(contIPNet) return nil } diff --git a/networking/util/link.go b/networking/util/link.go index 1c8f0b9..20b808b 100644 --- a/networking/util/link.go +++ b/networking/util/link.go @@ -45,16 +45,18 @@ func hash(s string) string { } // Should be in container netns +// TODO(eyakubovich): get rid of entropy and ask kernel to pick name via pattern func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (hostVeth, contVeth netlink.Link, err error) { - hostVethName := "rk" + hash(entropy)[:6] + // NetworkManager (recent versions) will ignore veth devices that start with "veth" + hostVethName := "veth" + hash(entropy)[:4] hostVeth, err = makeVeth(hostVethName, contVethName) if err != nil { err = fmt.Errorf("failed to make veth pair: %v", err) return } - if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil { - err = fmt.Errorf("failed to move veth to root netns: %v", err) + if err = netlink.LinkSetUp(hostVeth); err != nil { + err = fmt.Errorf("failed to set %q up: %v", hostVethName, err) return } @@ -65,7 +67,7 @@ func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (h } if err = netlink.LinkSetUp(contVeth); err != nil { - err = fmt.Errorf("failed to set eth0 up: %v", err) + err = fmt.Errorf("failed to set %q up: %v", contVethName, err) return } @@ -77,6 +79,11 @@ func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (h } } + if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil { + err = fmt.Errorf("failed to move veth to host netns: %v", err) + return + } + return } diff --git a/networking/util/net.go b/networking/util/net.go index 98c2310..1eac215 100644 --- a/networking/util/net.go +++ b/networking/util/net.go @@ -22,13 +22,13 @@ import ( type Net struct { Filename string - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` IPAlloc struct { Type string `json:"type,omitempty"` Subnet string `json:"subnet,omitempty"` - } `json:"ipAlloc,omitempty"` - Routes []string `json:"routes,omitempty"` + } `json:"ipAlloc,omitempty"` + Routes []string `json:"routes,omitempty"` } func LoadNet(path string, n interface{}) error { diff --git a/networking/util/route.go b/networking/util/route.go index 84fa31a..36b431f 100644 --- a/networking/util/route.go +++ b/networking/util/route.go @@ -34,3 +34,11 @@ func AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error { }) } +func AddHostRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error { + return netlink.RouteAdd(&netlink.Route{ + LinkIndex: dev.Attrs().Index, + Scope: netlink.SCOPE_HOST, + Dst: ipn, + Gw: gw, + }) +} diff --git a/networking/util/setns.go b/networking/util/setns.go index 2745957..57025b4 100644 --- a/networking/util/setns.go +++ b/networking/util/setns.go @@ -50,12 +50,14 @@ func WithNetNSPath(nspath string, f func(*os.File) error) error { if err != nil { return fmt.Errorf("Failed to open /proc/self/ns/net: %v", err) } + defer thisNS.Close() // switch to the container namespace ns, err := os.Open(nspath) if err != nil { return fmt.Errorf("Failed to open %v: %v", nspath, err) } + defer ns.Close() if err = SetNS(ns, syscall.CLONE_NEWNET); err != nil { return fmt.Errorf("Error switching to ns %v: %v", nspath, err) @@ -66,9 +68,5 @@ func WithNetNSPath(nspath string, f func(*os.File) error) error { } // switch back - if err = SetNS(thisNS, syscall.CLONE_NEWNET); err != nil { - return err - } - - return nil + return SetNS(thisNS, syscall.CLONE_NEWNET) }