net: Give the host end of veth an IP

The scheme of having a host veth not have an IP
and not be plugged into bridge does not work in
practice. This creates a true point-to-point link
between the container and the host.

networking/ipam/ipam.go

@@ -44,7 +44,7 @@ func ipAdd(ip net.IP, val uint) net.IP {
 	return net.IP(nip)
 }
 
-func allocIP(ipn *net.IPNet) (*net.IPNet, error) {
+func allocIP(ipn *net.IPNet) (net.IP, error) {
 	ones, bits := ipn.Mask.Size()
 	zeros := bits - ones
 	rng := (1 << uint(zeros)) - 2 // (reduce for gw, bcast)
@@ -55,11 +55,7 @@ func allocIP(ipn *net.IPNet) (*net.IPNet, error) {
 	}
 
 	offset := uint(n.Uint64() + 1)
-
-	return &net.IPNet{
-		IP: ipAdd(ipn.IP, offset),
-		Mask: ipn.Mask,
-	}, nil
+	return ipAdd(ipn.IP, offset), nil
 }
 
 func deallocIP(ip net.IP) error {
@@ -112,11 +108,15 @@ func AllocIP(contID types.UUID, netConf, ifName, args string) (*net.IPNet, net.I
 	}
 
 	if opts.ipRange != nil {
-		ipn, err := allocIP(opts.ipRange)
+		ip, err := allocIP(opts.ipRange)
 		if err != nil {
-			return nil, nil, fmt.Errorf("error allocating IP in %v: %v", ipn, err)
+			return nil, nil, fmt.Errorf("error allocating IP in %v: %v", opts.ipRange, err)
 		}
-		return ipn, nil, nil
+
+		return &net.IPNet{
+			IP:   ip,
+			Mask: opts.ipRange.Mask,
+		}, nil, nil
 	}
 
 	n := util.Net{}
@@ -126,25 +126,42 @@ func AllocIP(contID types.UUID, netConf, ifName, args string) (*net.IPNet, net.I
 
 	switch n.IPAlloc.Type {
 	case "static":
-		_, ipn, err := net.ParseCIDR(n.IPAlloc.Subnet)
+		_, rng, err := net.ParseCIDR(n.IPAlloc.Subnet)
 		if err != nil {
 			// TODO: cleanup
 			return nil, nil, fmt.Errorf("error parsing %q conf: ipAlloc.Subnet: %v")
 		}
 
-		ipn, err = allocIP(ipn)
+		ip, err := allocIP(rng)
 		if err != nil {
 			// TODO: cleanup
-			return nil, nil, fmt.Errorf("error allocating IP in %v: %v", ipn, err)
+			return nil, nil, fmt.Errorf("error allocating IP in %v: %v", rng, err)
 		}
 
-		return ipn, nil, nil
+		return &net.IPNet{
+			IP:   ip,
+			Mask: rng.Mask,
+		}, nil, nil
 
 	default:
 		return nil, nil, fmt.Errorf("unsupported IP allocation type")
 	}
 }
 
+// allocates a /31 for point-to-point links
+func AllocPtP(contID types.UUID, netConf, ifName, args string) ([2]net.IP, error) {
+	ipn, _, err := AllocIP(contID, netConf, ifName, args)
+	if err != nil {
+		return [2]net.IP{nil, nil}, err
+	}
+
+	mask := net.CIDRMask(31, 32)
+	first := ipn.IP.Mask(mask)
+	second := ipAdd(first, 1)
+
+	return [2]net.IP{first, second}, nil
+}
+
 func DeallocIP(contID types.UUID, netConf, ifName string, ipn *net.IPNet) error {
 	return nil
 }

networking/networking.go

@@ -30,7 +30,7 @@ import (
 
 const (
 	ifnamePattern = "eth%d"
-	selfNetNS = "/proc/self/ns/net"
+	selfNetNS     = "/proc/self/ns/net"
 )
 
 type activeNet struct {
@@ -141,7 +141,6 @@ func basicNetNS() (hostNS, contNS *os.File, err error) {
 	return
 }
 
-
 func (n *Networking) EnterHostNS() error {
 	return util.SetNS(n.hostNS, syscall.CLONE_NEWNET)
 }
@@ -163,7 +162,7 @@ func setupNets(contID types.UUID, netns string, plugins map[string]*NetPlugin, n
 		}
 
 		an := activeNet{
-			Net: nt,
+			Net:    nt,
 			ifName: fmt.Sprintf(ifnamePattern, i),
 		}
 

networking/plugins/veth/veth.go

@@ -36,7 +36,7 @@ func init() {
 }
 
 func cmdAdd(contID, netns, netConf, ifName, args string) error {
-	var hostVethName string
+	var hostVethName, contIPNet string
 
 	cid, err := types.NewUUID(contID)
 	if err != nil {
@@ -48,14 +48,21 @@ func cmdAdd(contID, netns, netConf, ifName, args string) error {
 		return fmt.Errorf("failed to load %q: %v", netConf, err)
 	}
 
-	ipn, gw, err := ipam.AllocIP(*cid, netConf, ifName, args)
+	ips, err := ipam.AllocPtP(*cid, netConf, ifName, args)
 	if err != nil {
 		return err
 	}
 
+	hostIP, contIP := ips[0], ips[1]
+
 	err = util.WithNetNSPath(netns, func(hostNS *os.File) error {
 		entropy := contID + ifName
 
+		ipn := &net.IPNet{
+			IP:   contIP,
+			Mask: net.CIDRMask(31, 32),
+		}
+
 		hostVeth, contVeth, err := util.SetupVeth(entropy, ifName, ipn, hostNS)
 		if err != nil {
 			return err
@@ -64,32 +71,44 @@ func cmdAdd(contID, netns, netConf, ifName, args string) error {
 		for _, r := range conf.Routes {
 			dst, err := util.ParseCIDR(r)
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to parse route %q: %v", r, err)
 			}
 
-			if err = util.AddRoute(dst, gw, contVeth); err != nil {
+			if err = util.AddRoute(dst, hostIP, contVeth); err != nil {
 				return fmt.Errorf("failed to add route %q: %v", dst, err)
 			}
 		}
 
 		hostVethName = hostVeth.Attrs().Name
-		return err
-	})
+		contIPNet = ipn.String()
 
-	// hostVeth moved namespaces and will have a new ifindex
-	hostVeth, err := netlink.LinkByName(hostVethName)
+		return nil
+	})
 	if err != nil {
-		return fmt.Errorf("failed to lookup %q: %v", hostVeth.Attrs().Name, err)
+		return err
 	}
 
-	// On the host we route traffic for the allocated IP to the container
-	ipn.Mask = net.CIDRMask(32, 32)
+	// hostVeth moved namespaces and may have a new ifindex
+	hostVeth, err := netlink.LinkByName(hostVethName)
+	if err != nil {
+		return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
+	}
 
-	if err = util.AddRoute(ipn, nil, hostVeth); err != nil {
+	ipn := &net.IPNet{
+		IP:   hostIP,
+		Mask: net.CIDRMask(31, 32),
+	}
+	addr := &netlink.Addr{ipn, ""}
+	if err = netlink.AddrAdd(hostVeth, addr); err != nil {
+		return fmt.Errorf("failed to add IP addr to veth: %v", err)
+	}
+
+	// dst happens to be the same as IP/net of host veth
+	if err = util.AddHostRoute(ipn, nil, hostVeth); err != nil && !os.IsExist(err) {
 		return fmt.Errorf("failed to add route on host: %v", err)
 	}
 
-	fmt.Print(ipn.String())
+	fmt.Print(contIPNet)
 
 	return nil
 }

networking/util/link.go

@@ -45,16 +45,18 @@ func hash(s string) string {
 }
 
 // Should be in container netns
+// TODO(eyakubovich): get rid of entropy and ask kernel to pick name via pattern
 func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (hostVeth, contVeth netlink.Link, err error) {
-	hostVethName := "rk" + hash(entropy)[:6]
+	// NetworkManager (recent versions) will ignore veth devices that start with "veth"
+	hostVethName := "veth" + hash(entropy)[:4]
 	hostVeth, err = makeVeth(hostVethName, contVethName)
 	if err != nil {
 		err = fmt.Errorf("failed to make veth pair: %v", err)
 		return
 	}
 
-	if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
-		err = fmt.Errorf("failed to move veth to root netns: %v", err)
+	if err = netlink.LinkSetUp(hostVeth); err != nil {
+		err = fmt.Errorf("failed to set %q up: %v", hostVethName, err)
 		return
 	}
 
@@ -65,7 +67,7 @@ func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (h
 	}
 
 	if err = netlink.LinkSetUp(contVeth); err != nil {
-		err = fmt.Errorf("failed to set eth0 up: %v", err)
+		err = fmt.Errorf("failed to set %q up: %v", contVethName, err)
 		return
 	}
 
@@ -77,6 +79,11 @@ func SetupVeth(entropy, contVethName string, ipn *net.IPNet, hostNS *os.File) (h
 		}
 	}
 
+	if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
+		err = fmt.Errorf("failed to move veth to host netns: %v", err)
+		return
+	}
+
 	return
 }
 

networking/util/net.go

@@ -22,13 +22,13 @@ import (
 
 type Net struct {
 	Filename string
-	Name     string   `json:"name,omitempty"`
-	Type     string   `json:"type,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Type     string `json:"type,omitempty"`
 	IPAlloc  struct {
 		Type   string `json:"type,omitempty"`
 		Subnet string `json:"subnet,omitempty"`
-	}                 `json:"ipAlloc,omitempty"`
-	Routes   []string `json:"routes,omitempty"`
+	} `json:"ipAlloc,omitempty"`
+	Routes []string `json:"routes,omitempty"`
 }
 
 func LoadNet(path string, n interface{}) error {

networking/util/route.go

@@ -34,3 +34,11 @@ func AddRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error {
 	})
 }
 
+func AddHostRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error {
+	return netlink.RouteAdd(&netlink.Route{
+		LinkIndex: dev.Attrs().Index,
+		Scope:     netlink.SCOPE_HOST,
+		Dst:       ipn,
+		Gw:        gw,
+	})
+}

networking/util/setns.go

@@ -50,12 +50,14 @@ func WithNetNSPath(nspath string, f func(*os.File) error) error {
 	if err != nil {
 		return fmt.Errorf("Failed to open /proc/self/ns/net: %v", err)
 	}
+	defer thisNS.Close()
 
 	// switch to the container namespace
 	ns, err := os.Open(nspath)
 	if err != nil {
 		return fmt.Errorf("Failed to open %v: %v", nspath, err)
 	}
+	defer ns.Close()
 
 	if err = SetNS(ns, syscall.CLONE_NEWNET); err != nil {
 		return fmt.Errorf("Error switching to ns %v: %v", nspath, err)
@@ -66,9 +68,5 @@ func WithNetNSPath(nspath string, f func(*os.File) error) error {
 	}
 
 	// switch back
-	if err = SetNS(thisNS, syscall.CLONE_NEWNET); err != nil {
-		return err
-	}
-
-	return nil
+	return SetNS(thisNS, syscall.CLONE_NEWNET)
 }