Merge pull request #4867 from crosbymichael/clean-shutdown

Cleanly shutdown docker

runtime/container.go

@@ -915,7 +915,6 @@ func (container *Container) Stop(seconds int) error {
 
 	// 1. Send a SIGTERM
 	if err := container.KillSig(15); err != nil {
-		utils.Debugf("Error sending kill SIGTERM: %s", err)
 		log.Print("Failed to send SIGTERM to the process, force killing")
 		if err := container.KillSig(9); err != nil {
 			return err

runtime/execdriver/driver.go

@@ -84,6 +84,7 @@ type Driver interface {
 	Name() string                                 // Driver name
 	Info(id string) Info                          // "temporary" hack (until we move state from core to plugins)
 	GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container.
+	Terminate(c *Command) error                   // kill it with fire
 }
 
 // Network settings of the container

runtime/execdriver/lxc/driver.go

@@ -204,6 +204,10 @@ func (d *driver) Kill(c *execdriver.Command, sig int) error {
 	return KillLxc(c.ID, sig)
 }
 
+func (d *driver) Terminate(c *execdriver.Command) error {
+	return KillLxc(c.ID, 9)
+}
+
 func (d *driver) version() string {
 	var (
 		version string

runtime/execdriver/native/driver.go

@@ -117,9 +117,39 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 }
 
 func (d *driver) Kill(p *execdriver.Command, sig int) error {
-	err := syscall.Kill(p.Process.Pid, syscall.Signal(sig))
+	return syscall.Kill(p.Process.Pid, syscall.Signal(sig))
+}
+
+func (d *driver) Terminate(p *execdriver.Command) error {
+	// lets check the start time for the process
+	started, err := d.readStartTime(p)
+	if err != nil {
+		// if we don't have the data on disk then we can assume the process is gone
+		// because this is only removed after we know the process has stopped
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	currentStartTime, err := system.GetProcessStartTime(p.Process.Pid)
+	if err != nil {
+		return err
+	}
+	if started == currentStartTime {
+		err = syscall.Kill(p.Process.Pid, 9)
+	}
 	d.removeContainerRoot(p.ID)
 	return err
+
+}
+
+func (d *driver) readStartTime(p *execdriver.Command) (string, error) {
+	data, err := ioutil.ReadFile(filepath.Join(d.root, p.ID, "start"))
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
 }
 
 func (d *driver) Info(id string) execdriver.Info {
@@ -235,9 +265,9 @@ type dockerStateWriter struct {
 	callback execdriver.StartCallback
 }
 
-func (d *dockerStateWriter) WritePid(pid int) error {
+func (d *dockerStateWriter) WritePid(pid int, started string) error {
 	d.c.ContainerPid = pid
-	err := d.dsw.WritePid(pid)
+	err := d.dsw.WritePid(pid, started)
 	if d.callback != nil {
 		d.callback(d.c)
 	}

runtime/runtime.go

@@ -174,6 +174,7 @@ func (runtime *Runtime) Register(container *Container) error {
 		if container.State.IsGhost() {
 			utils.Debugf("killing ghost %s", container.ID)
 
+			existingPid := container.State.Pid
 			container.State.SetGhost(false)
 			container.State.SetStopped(0)
 
@@ -181,9 +182,23 @@ func (runtime *Runtime) Register(container *Container) error {
 			// no ghost processes are left when docker dies
 			if container.ExecDriver == "" || strings.Contains(container.ExecDriver, "lxc") {
 				lxc.KillLxc(container.ID, 9)
-				if err := container.Unmount(); err != nil {
-					utils.Debugf("ghost unmount error %s", err)
+			} else {
+				// use the current driver and ensure that the container is dead x.x
+				cmd := &execdriver.Command{
+					ID: container.ID,
 				}
+				var err error
+				cmd.Process, err = os.FindProcess(existingPid)
+				if err != nil {
+					utils.Debugf("cannot find existing process for %d", existingPid)
+				}
+				runtime.execDriver.Terminate(cmd)
+			}
+			if err := container.Unmount(); err != nil {
+				utils.Debugf("ghost unmount error %s", err)
+			}
+			if err := container.ToDisk(); err != nil {
+				utils.Debugf("saving ghost state to disk %s", err)
 			}
 		}
 
@@ -778,8 +793,36 @@ func NewRuntimeFromDirectory(config *daemonconfig.Config, eng *engine.Engine) (*
 	return runtime, nil
 }
 
+func (runtime *Runtime) shutdown() error {
+	group := sync.WaitGroup{}
+	utils.Debugf("starting clean shutdown of all containers...")
+	for _, container := range runtime.List() {
+		c := container
+		if c.State.IsRunning() {
+			utils.Debugf("stopping %s", c.ID)
+			group.Add(1)
+
+			go func() {
+				defer group.Done()
+				if err := c.KillSig(15); err != nil {
+					utils.Debugf("kill 15 error for %s - %s", c.ID, err)
+				}
+				c.Wait()
+				utils.Debugf("container stopped %s", c.ID)
+			}()
+		}
+	}
+	group.Wait()
+
+	return nil
+}
+
 func (runtime *Runtime) Close() error {
 	errorsStrings := []string{}
+	if err := runtime.shutdown(); err != nil {
+		utils.Errorf("runtime.shutdown(): %s", err)
+		errorsStrings = append(errorsStrings, err.Error())
+	}
 	if err := portallocator.ReleaseAll(); err != nil {
 		utils.Errorf("portallocator.ReleaseAll(): %s", err)
 		errorsStrings = append(errorsStrings, err.Error())