From 59435f54d35018cbb578728d78c227729e283f3a Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Sun, 22 Feb 2015 17:24:22 -0800 Subject: [PATCH] WIP - Bridge refactoring Signed-off-by: Arnaud Porterie --- bridge/bridge.go | 253 +++++++++++++++++++++++++++++++++++++++-- bridge/interface.go | 23 ++++ bridge/setup.go | 51 +++++++++ bridge/setup_device.go | 54 +++++++++ bridge/setup_ipv4.go | 67 +++++++++++ bridge/utils.go | 73 ++++++++++++ namespace.go | 1 + 7 files changed, 514 insertions(+), 8 deletions(-) create mode 100644 bridge/interface.go create mode 100644 bridge/setup.go create mode 100644 bridge/setup_device.go create mode 100644 bridge/setup_ipv4.go create mode 100644 bridge/utils.go diff --git a/bridge/bridge.go b/bridge/bridge.go index 16fbe9a..5f47458 100644 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -6,26 +6,263 @@ import ( "github.com/docker/libnetwork" ) -const networkType = "bridgednetwork" +const ( + NetworkType = "simplebridge" + VethPrefix = "veth" +) -type bridgeConfiguration struct { - Subnet net.IPNet +type Configuration struct { + BridgeName string + AddressIPv4 *net.IPNet + AddressIPv6 *net.IPNet + FixedCIDR string + FixedCIDRv6 string + EnableIPv6 bool + EnableIPTables bool + EnableIPForwarding bool } func init() { - libnetwork.RegisterNetworkType(networkType, Create, &bridgeConfiguration{}) + libnetwork.RegisterNetworkType(NetworkType, Create, &Configuration{}) } -func Create(config *bridgeConfiguration) (libnetwork.Network, error) { - return &bridgeNetwork{Config: *config}, nil +/* + +func Create() + +- NewBridgeInterface(*Configuration) (*BridgeInterface, error) + . Issues LinkByName on config.BridgeName +- Create BridgeSetup instance with sequence of steps +- if !bridgeInterface.Exists() + . Add DeviceCreation (error if non-default name) + . Add AddressIPv4: set IPv4 (with automatic election if necessary) +- General case + . Add option EnableIPv6 if no IPv6 on bridge (disable_ipv6=0 + set IPv6) + . Verify configured addresses (with v4 and v6 updated in config) + . Add FixedCIDR v4: register subnet on IP Allocator + . Add FixedCIDR v6: register subnet on IP Allocator, route +- Add IPTables setup +- Add IPForward setup (depends on FixedCIDRv6) +- err := bridgeSetup.Apply() + +*/ + +func Create(config *Configuration) (libnetwork.Network, error) { + bridgeIntfc := NewInterface(config) + bridgeSetup := NewBridgeSetup(bridgeIntfc) + + // If the bridge interface doesn't exist, we need to start the setup steps + // by creating a new device and assigning it an IPv4 address. + bridgeAlreadyExists := bridgeIntfc.Exists() + if !bridgeAlreadyExists { + bridgeSetup.QueueStep(SetupDevice) + bridgeSetup.QueueStep(SetupBridgeIPv4) + } + + // Conditionnally queue setup steps depending on configuration values. + optSteps := []struct { + Condition bool + Fn SetupStep + }{ + // Enable IPv6 on the bridge if required. We do this even for a + // previously existing bridge, as it may be here from a previous + // installation where IPv6 wasn't supported yet and needs to be + // assigned an IPv6 link-local address. + {config.EnableIPv6, SetupBridgeIPv6}, + + // We ensure that the bridge has the expectedIPv4 and IPv6 addresses in + // the case of a previously existing device. + {bridgeAlreadyExists, SetupVerifyConfiguredAddresses}, + + // Setup the bridge to allocate containers IPv4 addresses in the + // specified subnet. + {config.FixedCIDR != "", SetupFixedCIDRv4}, + + // Setup the bridge to allocate containers global IPv6 addresses in the + // specified subnet. + {config.FixedCIDRv6 != "", SetupFixedCIDRv6}, + + // Setup IPTables. + {config.EnableIPTables, SetupIPTables}, + + // Setup IP forwarding. + {config.EnableIPForwarding, SetupIPForwarding}, + } + + for _, step := range optSteps { + if step.Condition { + bridgeSetup.QueueStep(step.Fn) + } + } + + // Apply the prepared list of steps, and abort at the first error. + bridgeSetup.QueueStep(SetupDeviceUp) + if err := bridgeSetup.Apply(); err != nil { + return nil, err + } + + return &bridgeNetwork{*config}, nil } +/* +func Create(config *Configuration) (libnetwork.Network, error) { + var ( + addrv4 netlink.Addr + addrsv6 []netlink.Addr + ) + + b := &bridgeNetwork{Config: *config} + if b.Config.BridgeName == "" { + b.Config.BridgeName = DefaultBridge + } + + link, err := netlink.LinkByName(b.Config.BridgeName) + if err != nil { + // The bridge interface doesn't exist, but we only attempt to create it + // if using the default name. + if b.Config.BridgeName != DefaultBridge { + return nil, err + } + + // Create the bridge interface. + if addrv4, addrsv6, err = createBridge(&b.Config); err != nil { + return nil, err + } + } else { + // The bridge interface exists: start by getting its configured + // addresses and verify if it matches the requested configuration. + addrv4, addrsv6, err = getInterfaceAddr(link) + if err != nil { + return nil, err + } + + if b.Config.AddressIPv4 != "" { + bridgeIP, _, err := net.ParseCIDR(b.Config.AddressIPv4) + if err != nil { + return nil, err + } + if !addrv4.IP.Equal(bridgeIP) { + return nil, fmt.Errorf("Bridge IP %s does not match requested configuration %s", addrv4.IP, bridgeIP) + } + } + + // A bridge might exist but not have any IPv6 addr associated with it + // yet (for example, an existing Docker installation that has only been + // used with IPv4 and docker0 already is set up). In that case, we can + // perform the bridge init for IPv6 here, else we will error out below + // if --ipv6=true. + if len(addrsv6) == 0 && config.EnableIPv6 { + if err := setupIPv6Bridge(iface, config); err != nil { + return nil, err + } + } + } + + return b, nil +} +*/ + +/* +func createBridge(config *Configuration) (netlink.Addr, []netlink.Addr, error) { + // Formats an error return with default values. + fmtError := func(format string, params ...interface{}) (netlink.Addr, []netlink.Addr, error) { + return netlink.Addr{}, nil, fmt.Errorf(format, params...) + } + + // Elect a subnet for the bridge interface. + bridgeIPNet, err := electBridgeNetwork(config) + if err != nil { + return fmtError("Failed to elect bridge network: %v", err) + } + log.Debugf("Creating bridge interface %q with network %s", config.BridgeName, bridgeIPNet) + + // We attempt to create the bridge, and ignore the returned error if it is + // already existing. + iface, err := createBridgeInterface(config.BridgeName) + if err != nil && !os.IsExist(err) { + return netlink.Addr{}, nil, err + } + + // Configure bridge IPv4. + if err := netlink.AddrAdd(iface, &netlink.Addr{bridgeIPNet, ""}); err != nil { + return fmtError("Failed to add address %s to bridge: %v", bridgeIPNet, err) + } + + // Configure bridge IPv6. + if config.EnableIPv6 { + if err := setupIPv6Bridge(iface, config); err != nil { + return fmtError("Failed to setup bridge IPv6: %v", err) + } + } + + // Up the bridge interface. + if err := netlink.LinkSetUp(iface); err != nil { + return fmtError("Failed to up network bridge: %v", err) + } + + if config.FixedCIDRv6 != "" { + dest, network, err := net.ParseCIDR(config.FixedCIDRv6) + if err != nil { + return fmtError("Invalid bridge fixed CIDR IPv6 %q: %v", config.FixedCIDRv6, err) + } + + // Set route to global IPv6 subnet + log.Infof("Adding route to IPv6 network %q via device %q", dest, iface) + if err := netlink.RouteAdd(&netlink.Route{Dst: network, LinkIndex: iface.Attrs().Index}); err != nil { + return fmtError("Could not add route to IPv6 network %q via device %q", config.FixedCIDRv6, iface) + } + } + + return getInterfaceAddrByName(config.BridgeName) +} + +func checkBridgeConfig(iface netlink.Link, config *Configuration) (netlink.Addr, []netlink.Addr, error) { + addrv4, addrsv6, err := getInterfaceAddr(iface) + if err != nil { + return netlink.Addr{}, nil, err + } + + // If config dictates a specific IP for the bridge, we have to check if it + // corresponds to reality. + if config.AddressIPv4 != "" { + bridgeIP, _, err := net.ParseCIDR(config.AddressIPv4) + if err != nil { + return netlink.Addr{}, nil, err + } + if !addrv4.IP.Equal(bridgeIP) { + return netlink.Addr{}, nil, fmt.Errorf("bridge ip (%s) does not match existing configuration %s", addrv4.IP, bridgeIP) + } + } + + return addrv4, addrsv6, nil +} + +func setupIPv6Bridge(iface netlink.Link, config *Configuration) error { + procFile := "/proc/sys/net/ipv6/conf/" + config.BridgeName + "/disable_ipv6" + if err := ioutil.WriteFile(procFile, []byte{'0', '\n'}, 0644); err != nil { + return fmt.Errorf("Unable to enable IPv6 addresses on bridge: %v", err) + } + + ip, net, err := net.ParseCIDR(config.AddressIPv6) + if err != nil { + return fmt.Errorf("Invalid bridge IPv6 address %q: %v", config.AddressIPv6, err) + } + + net.IP = ip + if err := netlink.AddrAdd(iface, &netlink.Addr{net, ""}); err != nil { + return fmt.Errorf("Failed to add address %s to bridge: %v", net, err) + } + + return nil +} +*/ + type bridgeNetwork struct { - Config bridgeConfiguration + Config Configuration } func (b *bridgeNetwork) Type() string { - return networkType + return NetworkType } func (b *bridgeNetwork) Link(name string) ([]*libnetwork.Interface, error) { diff --git a/bridge/interface.go b/bridge/interface.go new file mode 100644 index 0000000..7e6ba7f --- /dev/null +++ b/bridge/interface.go @@ -0,0 +1,23 @@ +package bridge + +import "github.com/vishvananda/netlink" + +type Interface struct { + Config *Configuration + Link netlink.Link +} + +func NewInterface(config *Configuration) *Interface { + i := &Interface{ + Config: config, + } + + // Attempt to find an existing bridge named with the specified name. + i.Link, _ = netlink.LinkByName(i.Config.BridgeName) + return i +} + +// Exists indicates if the existing bridge interface exists on the system. +func (i *Interface) Exists() bool { + return i.Link != nil +} diff --git a/bridge/setup.go b/bridge/setup.go new file mode 100644 index 0000000..ae05b54 --- /dev/null +++ b/bridge/setup.go @@ -0,0 +1,51 @@ +package bridge + +type SetupStep func(*Interface) error + +type BridgeSetup struct { + bridge *Interface + steps []SetupStep +} + +func NewBridgeSetup(b *Interface) *BridgeSetup { + return &BridgeSetup{bridge: b} +} + +func (b *BridgeSetup) Apply() error { + for _, fn := range b.steps { + if err := fn(b.bridge); err != nil { + return err + } + } + return nil +} + +func (b *BridgeSetup) QueueStep(step SetupStep) { + b.steps = append(b.steps, step) +} + +//---------------------------------------------------------------------------// + +func SetupBridgeIPv6(b *Interface) error { + return nil +} + +func SetupVerifyConfiguredAddresses(b *Interface) error { + return nil +} + +func SetupFixedCIDRv4(b *Interface) error { + return nil +} + +func SetupFixedCIDRv6(b *Interface) error { + return nil +} + +func SetupIPTables(b *Interface) error { + return nil +} + +func SetupIPForwarding(b *Interface) error { + return nil +} diff --git a/bridge/setup_device.go b/bridge/setup_device.go new file mode 100644 index 0000000..9b06f23 --- /dev/null +++ b/bridge/setup_device.go @@ -0,0 +1,54 @@ +package bridge + +import ( + "fmt" + "math/rand" + "net" + + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/parsers/kernel" + "github.com/vishvananda/netlink" +) + +const ( + DefaultBridgeName = "docker0" +) + +func SetupDevice(b *Interface) error { + // We only attempt to create the bridge when the requested device name is + // the default one. + if b.Config.BridgeName != DefaultBridgeName { + return fmt.Errorf("bridge device with non default name %q must be created manually", b.Config.BridgeName) + } + + // Set the Interface netlink.Bridge. + b.Link = &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{ + Name: b.Config.BridgeName, + }, + } + + // Only set the bridge's MAC address if the kernel version is > 3.3, as it + // was not supported before that. + kv, err := kernel.GetKernelVersion() + if err == nil && (kv.Kernel >= 3 && kv.Major >= 3) { + b.Link.Attrs().HardwareAddr = generateRandomMAC() + log.Debugf("Setting bridge mac address to %s", b.Link.Attrs().HardwareAddr) + } + + return netlink.LinkAdd(b.Link) +} + +func SetupDeviceUp(b *Interface) error { + return netlink.LinkSetUp(b.Link) +} + +func generateRandomMAC() net.HardwareAddr { + hw := make(net.HardwareAddr, 6) + for i := 0; i < 6; i++ { + hw[i] = byte(rand.Intn(255)) + } + hw[0] &^= 0x1 // clear multicast bit + hw[0] |= 0x2 // set local assignment bit (IEEE802) + return hw +} diff --git a/bridge/setup_ipv4.go b/bridge/setup_ipv4.go new file mode 100644 index 0000000..fd86604 --- /dev/null +++ b/bridge/setup_ipv4.go @@ -0,0 +1,67 @@ +package bridge + +import ( + "fmt" + "net" + + log "github.com/Sirupsen/logrus" + "github.com/vishvananda/netlink" +) + +var bridgeNetworks []*net.IPNet + +func init() { + // Here we don't follow the convention of using the 1st IP of the range for the gateway. + // This is to use the same gateway IPs as the /24 ranges, which predate the /16 ranges. + // In theory this shouldn't matter - in practice there's bound to be a few scripts relying + // on the internal addressing or other stupid things like that. + // They shouldn't, but hey, let's not break them unless we really have to. + for _, addr := range []string{ + "172.17.42.1/16", // Don't use 172.16.0.0/16, it conflicts with EC2 DNS 172.16.0.23 + "10.0.42.1/16", // Don't even try using the entire /8, that's too intrusive + "10.1.42.1/16", + "10.42.42.1/16", + "172.16.42.1/24", + "172.16.43.1/24", + "172.16.44.1/24", + "10.0.42.1/24", + "10.0.43.1/24", + "192.168.42.1/24", + "192.168.43.1/24", + "192.168.44.1/24", + } { + ip, net, err := net.ParseCIDR(addr) + if err != nil { + log.Errorf("Failed to parse address %s", addr) + continue + } + net.IP = ip + bridgeNetworks = append(bridgeNetworks, net) + } +} + +func SetupBridgeIPv4(b *Interface) error { + bridgeIPv4, err := electBridgeIPv4(b.Config) + if err != nil { + return err + } + + log.Debugf("Creating bridge interface %q with network %s", b.Config.BridgeName, bridgeIPv4) + return netlink.AddrAdd(b.Link, &netlink.Addr{bridgeIPv4, ""}) +} + +func electBridgeIPv4(config *Configuration) (*net.IPNet, error) { + // Use the requested IPv4 IP and mark when available. + if config.AddressIPv4 != nil { + return config.AddressIPv4, nil + } + + // Try to automatically elect appropriate brige IPv4 settings. + for _, n := range bridgeNetworks { + // TODO CheckNameserverOverlaps + // TODO CheckRouteOverlaps + return n, nil + } + + return nil, fmt.Errorf("Couldn't find an address range for interface %q", config.BridgeName) +} diff --git a/bridge/utils.go b/bridge/utils.go new file mode 100644 index 0000000..a4e6a11 --- /dev/null +++ b/bridge/utils.go @@ -0,0 +1,73 @@ +package bridge + +import "github.com/vishvananda/netlink" + +/* +func electBridgeNetwork(config *Configuration) (*net.IPNet, error) { + // Is a bridge IP is provided as part of the configuration, we only check + // its validity. + if config.AddressIPv4 != "" { + ip, network, err := net.ParseCIDR(config.AddressIPv4) + if err != nil { + return nil, err + } + network.IP = ip + return network, nil + } + + // No bridge IP was specified: we have to elect one ourselves from a set of + // predetermined networks. + for _, n := range bridgeNetworks { + // TODO CheckNameserverOverlaps + // TODO CheckRouteOverlaps + return n, nil + } + + return nil, fmt.Errorf("Couldn't find an address range for interface %q", config.BridgeName) +} + +func createBridgeInterface(name string) (netlink.Link, error) { + link := &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{ + Name: name, + }, + } + + // Only set the bridge's MAC address if the kernel version is > 3.3, as it + // was not supported before that. + kv, err := kernel.GetKernelVersion() + if err == nil && (kv.Kernel >= 3 && kv.Major >= 3) { + link.Attrs().HardwareAddr = generateRandomMAC() + log.Debugf("Setting bridge mac address to %s", link.Attrs().HardwareAddr) + } + + if err := netlink.LinkAdd(link); err != nil { + return nil, err + } + return netlink.LinkByName(name) +} +*/ + +func getInterfaceAddr(iface netlink.Link) (netlink.Addr, []netlink.Addr, error) { + v4addr, err := netlink.AddrList(iface, netlink.FAMILY_V4) + if err != nil { + return netlink.Addr{}, nil, err + } + + v6addr, err := netlink.AddrList(iface, netlink.FAMILY_V6) + if err != nil { + return netlink.Addr{}, nil, err + } + + // We only return the first IPv4 address, and the complete slice of IPv6 + // addresses. + return v4addr[0], v6addr, nil +} + +func getInterfaceAddrByName(ifaceName string) (netlink.Addr, []netlink.Addr, error) { + iface, err := netlink.LinkByName(ifaceName) + if err != nil { + return netlink.Addr{}, nil, err + } + return getInterfaceAddr(iface) +} diff --git a/namespace.go b/namespace.go index 99607a0..c09c350 100644 --- a/namespace.go +++ b/namespace.go @@ -14,6 +14,7 @@ func NewNamespace(path string) (Namespace, error) { } func (n *networkNamespace) AddInterface(i *Interface) error { + // TODO Open pipe, pass fd to child and write serialized Interface on it. if err := reexec(reexecMoveInterface, i.SrcName, i.DstName); err != nil { return err }