Add LeaveAll support

Currently container can join one endpoint when it is started.
More endpoints can be attached at a later point in time. But
when that happens this attachment should only have meaning
only as long as the container is alive. The attachment should
lose it's meaning when the container goes away. Cuurently there
is no way for the container management code to tell libnetwork
to detach the container from all attached endpoints. This PR
provides an additional API `LeaveAll` which adds this
functionality,

To facilitate this and make the sanbox lifecycle consistent
some slight changes have been made to the behavior of sandbox
management code. The sandbox is no longer destroyed when the
last endpoint is detached from the container. Instead the sandbox
ie kept alive and can only be destroyed with a `LeaveAll` call.
This gives better control of sandbox lifecycle by the container
management code and the sandbox doesn't get destroyed from under
the carpet while the container is still using it.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

sandboxdata.go

@@ -2,6 +2,7 @@ package libnetwork
 
 import (
 	"container/heap"
+	"fmt"
 	"sync"
 
 	"github.com/Sirupsen/logrus"
@@ -48,13 +49,9 @@ func (eh *epHeap) Pop() interface{} {
 
 func (s *sandboxData) updateGateway(ep *endpoint) error {
 	sb := s.sandbox()
-	if err := sb.UnsetGateway(); err != nil {
-		return err
-	}
 
-	if err := sb.UnsetGatewayIPv6(); err != nil {
-		return err
-	}
+	sb.UnsetGateway()
+	sb.UnsetGatewayIPv6()
 
 	if ep == nil {
 		return nil
@@ -65,11 +62,11 @@ func (s *sandboxData) updateGateway(ep *endpoint) error {
 	ep.Unlock()
 
 	if err := sb.SetGateway(joinInfo.gw); err != nil {
-		return err
+		return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
 	}
 
 	if err := sb.SetGatewayIPv6(joinInfo.gw6); err != nil {
-		return err
+		return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
 	}
 
 	return nil
@@ -93,7 +90,7 @@ func (s *sandboxData) addEndpoint(ep *endpoint) error {
 		}
 
 		if err := sb.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {
-			return err
+			return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)
 		}
 	}
 
@@ -101,7 +98,7 @@ func (s *sandboxData) addEndpoint(ep *endpoint) error {
 		// Set up non-interface routes.
 		for _, r := range ep.joinInfo.StaticRoutes {
 			if err := sb.AddStaticRoute(r); err != nil {
-				return err
+				return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err)
 			}
 		}
 	}
@@ -117,14 +114,10 @@ func (s *sandboxData) addEndpoint(ep *endpoint) error {
 		}
 	}
 
-	s.Lock()
-	s.refCnt++
-	s.Unlock()
-
 	return nil
 }
 
-func (s *sandboxData) rmEndpoint(ep *endpoint) int {
+func (s *sandboxData) rmEndpoint(ep *endpoint) {
 	ep.Lock()
 	joinInfo := ep.joinInfo
 	ep.Unlock()
@@ -171,17 +164,6 @@ func (s *sandboxData) rmEndpoint(ep *endpoint) int {
 	if highEpBefore != highEpAfter {
 		s.updateGateway(highEpAfter)
 	}
-
-	s.Lock()
-	s.refCnt--
-	refCnt := s.refCnt
-	s.Unlock()
-
-	if refCnt == 0 {
-		s.sandbox().Destroy()
-	}
-
-	return refCnt
 }
 
 func (s *sandboxData) sandbox() sandbox.Sandbox {
@@ -199,7 +181,7 @@ func (c *controller) sandboxAdd(key string, create bool, ep *endpoint) (sandbox.
 	if !ok {
 		sb, err := sandbox.NewSandbox(key, create)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to create new sandbox: %v", err)
 		}
 
 		sData = &sandboxData{
@@ -225,11 +207,7 @@ func (c *controller) sandboxRm(key string, ep *endpoint) {
 	sData := c.sandboxes[key]
 	c.Unlock()
 
-	if sData.rmEndpoint(ep) == 0 {
-		c.Lock()
-		delete(c.sandboxes, key)
-		c.Unlock()
-	}
+	sData.rmEndpoint(ep)
 }
 
 func (c *controller) sandboxGet(key string) sandbox.Sandbox {
@@ -243,3 +221,32 @@ func (c *controller) sandboxGet(key string) sandbox.Sandbox {
 
 	return sData.sandbox()
 }
+
+func (c *controller) LeaveAll(id string) error {
+	c.Lock()
+	sData, ok := c.sandboxes[sandbox.GenerateKey(id)]
+	c.Unlock()
+
+	if !ok {
+		c.Unlock()
+		return fmt.Errorf("could not find sandbox for container id %s", id)
+	}
+
+	sData.Lock()
+	eps := make([]*endpoint, len(sData.endpoints))
+	for i, ep := range sData.endpoints {
+		eps[i] = ep
+	}
+	sData.Unlock()
+
+	for _, ep := range eps {
+		if err := ep.Leave(id); err != nil {
+			logrus.Warnf("Failed leaving endpoint id %s: %v\n", ep.ID(), err)
+		}
+	}
+
+	sData.sandbox().Destroy()
+	delete(c.sandboxes, sandbox.GenerateKey(id))
+
+	return nil
+}