Add --userland-proxy daemon flag

The `--userland-proxy` daemon flag makes it possible to rely on hairpin
NAT and additional iptables routes instead of userland proxy for port
publishing and inter-container communication.

Usage of the userland proxy remains the default as hairpin NAT is
unsupported by older kernels.

Signed-off-by: Arnaud Porterie <arnaud.porterie@docker.com>

daemon/networkdriver/bridge/driver.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -83,6 +84,7 @@ var (
 	gatewayIPv6       net.IP
 	portMapper        *portmapper.PortMapper
 	once              sync.Once
+	hairpinMode       bool
 
 	defaultBindingIP  = net.ParseIP("0.0.0.0")
 	currentInterfaces = ifaces{c: make(map[string]*networkInterface)}
@@ -100,6 +102,7 @@ type Config struct {
 	EnableIptables              bool
 	EnableIpForward             bool
 	EnableIpMasq                bool
+	EnableUserlandProxy         bool
 	DefaultIp                   net.IP
 	Iface                       string
 	IP                          string
@@ -131,6 +134,8 @@ func InitDriver(config *Config) error {
 		defaultBindingIP = config.DefaultIp
 	}
 
+	hairpinMode = !config.EnableUserlandProxy
+
 	bridgeIface = config.Iface
 	usingDefaultBridge := false
 	if bridgeIface == "" {
@@ -243,39 +248,46 @@ func InitDriver(config *Config) error {
 	if config.EnableIpForward {
 		// Enable IPv4 forwarding
 		if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte{'1', '\n'}, 0644); err != nil {
-			logrus.Warnf("WARNING: unable to enable IPv4 forwarding: %s\n", err)
+			logrus.Warnf("Unable to enable IPv4 forwarding: %v", err)
 		}
 
 		if config.FixedCIDRv6 != "" {
 			// Enable IPv6 forwarding
 			if err := ioutil.WriteFile("/proc/sys/net/ipv6/conf/default/forwarding", []byte{'1', '\n'}, 0644); err != nil {
-				logrus.Warnf("WARNING: unable to enable IPv6 default forwarding: %s\n", err)
+				logrus.Warnf("Unable to enable IPv6 default forwarding: %v", err)
 			}
 			if err := ioutil.WriteFile("/proc/sys/net/ipv6/conf/all/forwarding", []byte{'1', '\n'}, 0644); err != nil {
-				logrus.Warnf("WARNING: unable to enable IPv6 all forwarding: %s\n", err)
+				logrus.Warnf("Unable to enable IPv6 all forwarding: %v", err)
 			}
 		}
 	}
 
+	if hairpinMode {
+		// Enable loopback adresses routing
+		sysPath := filepath.Join("/proc/sys/net/ipv4/conf", bridgeIface, "route_localnet")
+		if err := ioutil.WriteFile(sysPath, []byte{'1', '\n'}, 0644); err != nil {
+			logrus.Warnf("Unable to enable local routing for hairpin mode: %v", err)
+		}
+	}
+
 	// We can always try removing the iptables
 	if err := iptables.RemoveExistingChain("DOCKER", iptables.Nat); err != nil {
 		return err
 	}
 
 	if config.EnableIptables {
-		_, err := iptables.NewChain("DOCKER", bridgeIface, iptables.Nat)
+		_, err := iptables.NewChain("DOCKER", bridgeIface, iptables.Nat, hairpinMode)
 		if err != nil {
 			return err
 		}
 		// call this on Firewalld reload
-		iptables.OnReloaded(func() { iptables.NewChain("DOCKER", bridgeIface, iptables.Nat) })
-
-		chain, err := iptables.NewChain("DOCKER", bridgeIface, iptables.Filter)
+		iptables.OnReloaded(func() { iptables.NewChain("DOCKER", bridgeIface, iptables.Nat, hairpinMode) })
+		chain, err := iptables.NewChain("DOCKER", bridgeIface, iptables.Filter, hairpinMode)
 		if err != nil {
 			return err
 		}
 		// call this on Firewalld reload
-		iptables.OnReloaded(func() { iptables.NewChain("DOCKER", bridgeIface, iptables.Filter) })
+		iptables.OnReloaded(func() { iptables.NewChain("DOCKER", bridgeIface, iptables.Filter, hairpinMode) })
 
 		portMapper.SetIptablesChain(chain)
 	}
@@ -374,6 +386,18 @@ func setupIPTables(addr net.Addr, icc, ipmasq bool) error {
 		}
 	}
 
+	// In hairpin mode, masquerade traffic from localhost
+	if hairpinMode {
+		masqueradeArgs := []string{"-t", "nat", "-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}
+		if !iptables.Exists(iptables.Filter, "POSTROUTING", masqueradeArgs...) {
+			if output, err := iptables.Raw(append([]string{"-I", "POSTROUTING"}, masqueradeArgs...)...); err != nil {
+				return fmt.Errorf("Unable to masquerade local traffic: %s", err)
+			} else if len(output) != 0 {
+				return fmt.Errorf("Error iptables masquerade local traffic: %s", output)
+			}
+		}
+	}
+
 	// Accept all non-intercontainer outgoing packets
 	outgoingArgs := []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}
 	if !iptables.Exists(iptables.Filter, "FORWARD", outgoingArgs...) {
@@ -637,6 +661,7 @@ func Allocate(id, requestedMac, requestedIP, requestedIPv6 string) (*network.Set
 		Bridge:               bridgeIface,
 		IPPrefixLen:          maskSize,
 		LinkLocalIPv6Address: localIPv6.String(),
+		HairpinMode:          hairpinMode,
 	}
 
 	if globalIPv6Network != nil {
@@ -722,7 +747,7 @@ func AllocatePort(id string, port nat.Port, binding nat.PortBinding) (nat.PortBi
 		return nat.PortBinding{}, err
 	}
 	for i := 0; i < MaxAllocatedPortAttempts; i++ {
-		if host, err = portMapper.Map(container, ip, hostPort); err == nil {
+		if host, err = portMapper.Map(container, ip, hostPort, !hairpinMode); err == nil {
 			break
 		}
 		// There is no point in immediately retrying to map an explicitly

daemon/networkdriver/bridge/driver_test.go

@@ -177,7 +177,7 @@ func TestLinkContainers(t *testing.T) {
 	}
 
 	bridgeIface = "lo"
-	if _, err := iptables.NewChain("DOCKER", bridgeIface, iptables.Filter); err != nil {
+	if _, err := iptables.NewChain("DOCKER", bridgeIface, iptables.Filter, false); err != nil {
 		t.Fatal(err)
 	}
 

daemon/networkdriver/portmapper/mapper.go

@@ -51,7 +51,7 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.Chain) {
 	pm.chain = c
 }
 
-func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host net.Addr, err error) {
+func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
 	pm.lock.Lock()
 	defer pm.lock.Unlock()
 
@@ -59,7 +59,6 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 		m                 *mapping
 		proto             string
 		allocatedHostPort int
-		proxy             UserlandProxy
 	)
 
 	switch container.(type) {
@@ -75,7 +74,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 			container: container,
 		}
 
-		proxy = NewProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
+		if useProxy {
+			m.userlandProxy = NewProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
+		}
 	case *net.UDPAddr:
 		proto = "udp"
 		if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil {
@@ -88,7 +89,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 			container: container,
 		}
 
-		proxy = NewProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
+		if useProxy {
+			m.userlandProxy = NewProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
+		}
 	default:
 		return nil, ErrUnknownBackendAddressType
 	}
@@ -112,7 +115,9 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 
 	cleanup := func() error {
 		// need to undo the iptables rules before we return
-		proxy.Stop()
+		if m.userlandProxy != nil {
+			m.userlandProxy.Stop()
+		}
 		pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
 		if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
 			return err
@@ -121,13 +126,15 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int) (host
 		return nil
 	}
 
-	if err := proxy.Start(); err != nil {
-		if err := cleanup(); err != nil {
-			return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
+	if m.userlandProxy != nil {
+		if err := m.userlandProxy.Start(); err != nil {
+			if err := cleanup(); err != nil {
+				return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
+			}
+			return nil, err
 		}
-		return nil, err
 	}
-	m.userlandProxy = proxy
+
 	pm.currentMappings[key] = m
 	return m.host, nil
 }
@@ -154,7 +161,9 @@ func (pm *PortMapper) Unmap(host net.Addr) error {
 		return ErrPortNotMapped
 	}
 
-	data.userlandProxy.Stop()
+	if data.userlandProxy != nil {
+		data.userlandProxy.Stop()
+	}
 
 	delete(pm.currentMappings, key)
 

daemon/networkdriver/portmapper/mapper_test.go

@@ -44,22 +44,22 @@ func TestMapPorts(t *testing.T) {
 		return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
 	}
 
-	if host, err := pm.Map(srcAddr1, dstIp1, 80); err != nil {
+	if host, err := pm.Map(srcAddr1, dstIp1, 80, true); err != nil {
 		t.Fatalf("Failed to allocate port: %s", err)
 	} else if !addrEqual(dstAddr1, host) {
 		t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
 			dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
 	}
 
-	if _, err := pm.Map(srcAddr1, dstIp1, 80); err == nil {
+	if _, err := pm.Map(srcAddr1, dstIp1, 80, true); err == nil {
 		t.Fatalf("Port is in use - mapping should have failed")
 	}
 
-	if _, err := pm.Map(srcAddr2, dstIp1, 80); err == nil {
+	if _, err := pm.Map(srcAddr2, dstIp1, 80, true); err == nil {
 		t.Fatalf("Port is in use - mapping should have failed")
 	}
 
-	if _, err := pm.Map(srcAddr2, dstIp2, 80); err != nil {
+	if _, err := pm.Map(srcAddr2, dstIp2, 80, true); err != nil {
 		t.Fatalf("Failed to allocate port: %s", err)
 	}
 
@@ -127,14 +127,14 @@ func TestMapAllPortsSingleInterface(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		start, end := pm.Allocator.Begin, pm.Allocator.End
 		for i := start; i < end; i++ {
-			if host, err = pm.Map(srcAddr1, dstIp1, 0); err != nil {
+			if host, err = pm.Map(srcAddr1, dstIp1, 0, true); err != nil {
 				t.Fatal(err)
 			}
 
 			hosts = append(hosts, host)
 		}
 
-		if _, err := pm.Map(srcAddr1, dstIp1, start); err == nil {
+		if _, err := pm.Map(srcAddr1, dstIp1, start, true); err == nil {
 			t.Fatalf("Port %d should be bound but is not", start)
 		}