diff --git a/.gitignore b/.gitignore index fc4ca5da3..d55740421 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ docker/docker dockerd/dockerd .*.swp a.out +*.orig diff --git a/Vagrantfile b/Vagrantfile index dcc3e50bd..c823a3958 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,11 +7,11 @@ Vagrant::Config.run do |config| # please see the online documentation at vagrantup.com. # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = "quantal64" + config.vm.box = "quantal64_3.5.0-25" # The url from where the 'config.vm.box' box will be fetched if it # doesn't already exist on the user's system. - config.vm.box_url = "http://unworkable.org/~niallo/quantal64.box" + config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box" # Boot with a GUI so you can see the screen. (Default is headless) # config.vm.boot_mode = :gui diff --git a/client/client.go b/client/client.go index 4c4ea1c5e..814aed2f1 100644 --- a/client/client.go +++ b/client/client.go @@ -111,6 +111,7 @@ func InteractiveMode(scripts ...string) error { if err != nil { return err } + defer os.Remove(rcfile.Name()) io.WriteString(rcfile, "enable -n help\n") os.Setenv("PATH", tmp+":"+os.Getenv("PATH")) os.Setenv("PS1", "\\h docker> ") diff --git a/container.go b/container.go index 530c08bc3..c80129e2a 100644 --- a/container.go +++ b/container.go @@ -3,6 +3,7 @@ package docker import ( "encoding/json" "errors" + "github.com/dotcloud/docker/fs" "github.com/kr/pty" "io" "io/ioutil" @@ -31,8 +32,9 @@ type Container struct { Args []string Config *Config - Filesystem *Filesystem + Mountpoint *fs.Mountpoint State *State + Image string network *NetworkInterface networkManager *NetworkManager @@ -51,12 +53,13 @@ type Container struct { } type Config struct { - Hostname string - User string - Ram int64 - Ports []int - Tty bool // Attach standard streams to a tty, including stdin if it is not closed. - OpenStdin bool // Open stdin + Hostname string + User string + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + Ports []int + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin } type NetworkSettings struct { @@ -66,7 +69,11 @@ type NetworkSettings struct { PortMapping map[string]string } -func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netManager *NetworkManager) (*Container, error) { +func createContainer(id string, root string, command string, args []string, image *fs.Image, config *Config, netManager *NetworkManager) (*Container, error) { + mountpoint, err := image.Mountpoint(path.Join(root, "rootfs"), path.Join(root, "rw")) + if err != nil { + return nil, err + } container := &Container{ Id: id, Root: root, @@ -74,7 +81,8 @@ func createContainer(id string, root string, command string, args []string, laye Path: command, Args: args, Config: config, - Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), + Image: image.Id, + Mountpoint: mountpoint, State: newState(), networkManager: netManager, NetworkSettings: &NetworkSettings{}, @@ -105,31 +113,39 @@ func createContainer(id string, root string, command string, args []string, laye container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - if err := container.Filesystem.createMountPoints(); err != nil { - return nil, err - } if err := container.save(); err != nil { return nil, err } return container, nil } -func loadContainer(containerPath string, netManager *NetworkManager) (*Container, error) { +func loadContainer(store *fs.Store, containerPath string, netManager *NetworkManager) (*Container, error) { data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) if err != nil { return nil, err } + mountpoint, err := store.FetchMountpoint( + path.Join(containerPath, "rootfs"), + path.Join(containerPath, "rw"), + ) + if err != nil { + return nil, err + } else if mountpoint == nil { + return nil, errors.New("Couldn't load container: unregistered mountpoint.") + } container := &Container{ stdout: newWriteBroadcaster(), stderr: newWriteBroadcaster(), lxcConfigPath: path.Join(containerPath, "config.lxc"), networkManager: netManager, NetworkSettings: &NetworkSettings{}, + Mountpoint: mountpoint, } // Load container settings if err := json.Unmarshal(data, container); err != nil { return nil, err } + // Setup logging of stdout and stderr to disk if stdoutLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { return nil, err @@ -144,10 +160,6 @@ func loadContainer(containerPath string, netManager *NetworkManager) (*Container container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - // Create mountpoints - if err := container.Filesystem.createMountPoints(); err != nil { - return nil, err - } if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() } else { @@ -297,7 +309,7 @@ func (container *Container) start() error { } func (container *Container) Start() error { - if err := container.Filesystem.EnsureMounted(); err != nil { + if err := container.Mountpoint.EnsureMounted(); err != nil { return err } if err := container.allocateNetwork(); err != nil { @@ -438,7 +450,7 @@ func (container *Container) monitor() { } container.stdout.Close() container.stderr.Close() - if err := container.Filesystem.Umount(); err != nil { + if err := container.Mountpoint.Umount(); err != nil { log.Printf("%v: Failed to umount filesystem: %v", container.Id, err) } diff --git a/container_test.go b/container_test.go index 8c187ecb9..18beeba25 100644 --- a/container_test.go +++ b/container_test.go @@ -1,48 +1,93 @@ package docker import ( + "bufio" "fmt" + "github.com/dotcloud/docker/fs" "io" "io/ioutil" + "math/rand" + "os" "sort" "strings" "testing" "time" ) -func TestStart(t *testing.T) { +func TestCommitRun(t *testing.T) { docker, err := newTestDocker() if err != nil { t.Fatal(err) } - container, err := docker.Create( - "start_test", - "ls", - []string{"-al"}, - []string{testLayerPath}, + defer nuke(docker) + container1, err := docker.Create( + "precommit_test", + "/bin/sh", + []string{"-c", "echo hello > /world"}, + GetTestImage(docker), &Config{ - Ram: 33554432, + Memory: 33554432, }, ) if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer docker.Destroy(container1) - if container.State.Running { + if container1.State.Running { t.Errorf("Container shouldn't be running") } - if err := container.Start(); err != nil { + if err := container1.Run(); err != nil { t.Fatal(err) } - container.Wait() - if container.State.Running { + if container1.State.Running { t.Errorf("Container shouldn't be running") } - // We should be able to call Wait again - container.Wait() - if container.State.Running { - t.Errorf("Container shouldn't be running") + + // FIXME: freeze the container before copying it to avoid data corruption? + rwTar, err := fs.Tar(container1.Mountpoint.Rw, fs.Uncompressed) + if err != nil { + t.Error(err) + } + // Create a new image from the container's base layers + a new layer from container changes + parentImg, err := docker.Store.Get(container1.Image) + if err != nil { + t.Error(err) + } + + img, err := docker.Store.Create(rwTar, parentImg, "test_commitrun", "unit test commited image") + if err != nil { + t.Error(err) + } + + // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world + + container2, err := docker.Create( + "postcommit_test", + "cat", + []string{"/world"}, + img, + &Config{ + Memory: 33554432, + }, + ) + if err != nil { + t.Fatal(err) + } + defer docker.Destroy(container2) + + stdout, err := container2.StdoutPipe() + stderr, err := container2.StderrPipe() + if err := container2.Start(); err != nil { + t.Fatal(err) + } + container2.Wait() + output, err := ioutil.ReadAll(stdout) + output2, err := ioutil.ReadAll(stderr) + stdout.Close() + stderr.Close() + if string(output) != "hello\n" { + t.Fatalf("\nout: %s\nerr: %s\n", string(output), string(output2)) } } @@ -51,13 +96,14 @@ func TestRun(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "run_test", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ - Ram: 33554432, + Memory: 33554432, }, ) if err != nil { @@ -81,11 +127,12 @@ func TestOutput(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "output_test", "echo", []string{"-n", "foobar"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -106,11 +153,12 @@ func TestKill(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "stop_test", "cat", []string{"/dev/zero"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -148,12 +196,13 @@ func TestExitCode(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) trueContainer, err := docker.Create( "exit_test_1", "/bin/true", []string{""}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -168,7 +217,7 @@ func TestExitCode(t *testing.T) { "exit_test_2", "/bin/false", []string{""}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -193,11 +242,12 @@ func TestRestart(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "restart_test", "echo", []string{"-n", "foobar"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -227,11 +277,12 @@ func TestRestartStdin(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "restart_stdin_test", "cat", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ OpenStdin: true, }, @@ -276,13 +327,14 @@ func TestUser(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) // Default user must be root container, err := docker.Create( "user_default", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -302,7 +354,7 @@ func TestUser(t *testing.T) { "user_root", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "root", }, @@ -324,7 +376,7 @@ func TestUser(t *testing.T) { "user_uid0", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "0", }, @@ -346,7 +398,7 @@ func TestUser(t *testing.T) { "user_uid1", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "1", }, @@ -356,8 +408,10 @@ func TestUser(t *testing.T) { } defer docker.Destroy(container) output, err = container.Output() - if err != nil || container.State.ExitCode != 0 { + if err != nil { t.Fatal(err) + } else if container.State.ExitCode != 0 { + t.Fatalf("Container exit code is invalid: %d\nOutput:\n%s\n", container.State.ExitCode, output) } if !strings.Contains(string(output), "uid=1(daemon) gid=1(daemon)") { t.Error(string(output)) @@ -368,7 +422,7 @@ func TestUser(t *testing.T) { "user_daemon", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "daemon", }, @@ -391,12 +445,13 @@ func TestMultipleContainers(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container1, err := docker.Create( "container1", "cat", []string{"/dev/zero"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -408,7 +463,7 @@ func TestMultipleContainers(t *testing.T) { "container2", "cat", []string{"/dev/zero"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -447,11 +502,12 @@ func TestStdin(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "stdin_test", "cat", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ OpenStdin: true, }, @@ -482,11 +538,12 @@ func TestTty(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "tty_test", "cat", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ OpenStdin: true, }, @@ -517,11 +574,12 @@ func TestEnv(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "env_test", "/usr/bin/env", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -561,17 +619,71 @@ func TestEnv(t *testing.T) { } } +func grepFile(t *testing.T, path string, pattern string) { + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + r := bufio.NewReader(f) + var ( + line string + ) + err = nil + for err == nil { + line, err = r.ReadString('\n') + if strings.Contains(line, pattern) == true { + return + } + } + t.Fatalf("grepFile: pattern \"%s\" not found in \"%s\"", pattern, path) +} + +func TestLXCConfig(t *testing.T) { + docker, err := newTestDocker() + if err != nil { + t.Fatal(err) + } + defer nuke(docker) + // Memory is allocated randomly for testing + rand.Seed(time.Now().UTC().UnixNano()) + memMin := 33554432 + memMax := 536870912 + mem := memMin + rand.Intn(memMax-memMin) + container, err := docker.Create( + "config_test", + "/bin/true", + []string{}, + GetTestImage(docker), + &Config{ + Hostname: "foobar", + Memory: int64(mem), + }, + ) + if err != nil { + t.Fatal(err) + } + defer docker.Destroy(container) + container.generateLXCConfig() + grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") + grepFile(t, container.lxcConfigPath, + fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) + grepFile(t, container.lxcConfigPath, + fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2)) +} + func BenchmarkRunSequencial(b *testing.B) { docker, err := newTestDocker() if err != nil { b.Fatal(err) } + defer nuke(docker) for i := 0; i < b.N; i++ { container, err := docker.Create( fmt.Sprintf("bench_%v", i), "echo", []string{"-n", "foo"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -596,6 +708,7 @@ func BenchmarkRunParallel(b *testing.B) { if err != nil { b.Fatal(err) } + defer nuke(docker) var tasks []chan error @@ -607,7 +720,7 @@ func BenchmarkRunParallel(b *testing.B) { fmt.Sprintf("bench_%v", i), "echo", []string{"-n", "foo"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { diff --git a/docker.go b/docker.go index 67b67f7cd..8e8b19082 100644 --- a/docker.go +++ b/docker.go @@ -3,6 +3,7 @@ package docker import ( "container/list" "fmt" + "github.com/dotcloud/docker/fs" "io/ioutil" "log" "os" @@ -15,6 +16,7 @@ type Docker struct { repository string containers *list.List networkManager *NetworkManager + Store *fs.Store } func (docker *Docker) List() []*Container { @@ -47,12 +49,13 @@ func (docker *Docker) Exists(id string) bool { return docker.Get(id) != nil } -func (docker *Docker) Create(id string, command string, args []string, layers []string, config *Config) (*Container, error) { +func (docker *Docker) Create(id string, command string, args []string, image *fs.Image, config *Config) (*Container, error) { if docker.Exists(id) { return nil, fmt.Errorf("Container %v already exists", id) } root := path.Join(docker.repository, id) - container, err := createContainer(id, root, command, args, layers, config, docker.networkManager) + + container, err := createContainer(id, root, command, args, image, config, docker.networkManager) if err != nil { return nil, err } @@ -69,10 +72,14 @@ func (docker *Docker) Destroy(container *Container) error { if err := container.Stop(); err != nil { return err } - if container.Filesystem.IsMounted() { - if err := container.Filesystem.Umount(); err != nil { + if container.Mountpoint.Mounted() { + if err := container.Mountpoint.Umount(); err != nil { log.Printf("Unable to umount container %v: %v", container.Id, err) } + + if err := container.Mountpoint.Deregister(); err != nil { + log.Printf("Unable to deregiser mountpoint %v: %v", container.Mountpoint.Root, err) + } } if err := os.RemoveAll(container.Root); err != nil { log.Printf("Unable to remove filesystem for %v: %v", container.Id, err) @@ -87,7 +94,7 @@ func (docker *Docker) restore() error { return err } for _, v := range dir { - container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkManager) + container, err := loadContainer(docker.Store, path.Join(docker.repository, v.Name()), docker.networkManager) if err != nil { log.Printf("Failed to load container %v: %v", v.Name(), err) continue @@ -102,14 +109,20 @@ func New() (*Docker, error) { } func NewFromDirectory(root string) (*Docker, error) { + store, err := fs.New(path.Join(root, "images")) + if err != nil { + return nil, err + } netManager, err := newNetworkManager(networkBridgeIface) if err != nil { return nil, err } + docker := &Docker{ root: root, repository: path.Join(root, "containers"), containers: list.New(), + Store: store, networkManager: netManager, } diff --git a/docker_test.go b/docker_test.go index d786da950..3f51e8d84 100644 --- a/docker_test.go +++ b/docker_test.go @@ -1,13 +1,28 @@ package docker import ( + "github.com/dotcloud/docker/fs" + "io" "io/ioutil" "log" "os" "testing" ) -const testLayerPath string = "/var/lib/docker/images/docker-ut" +const testLayerPath string = "/var/lib/docker/docker-ut.tar" + +func nuke(docker *Docker) error { + return os.RemoveAll(docker.root) +} + +func layerArchive(tarfile string) (io.Reader, error) { + // FIXME: need to close f somewhere + f, err := os.Open(tarfile) + if err != nil { + return nil, err + } + return f, nil +} func init() { // Hack to run sys init during unit testing @@ -21,6 +36,7 @@ func init() { panic(err) } log.Fatalf("Unit test base image not found. Please fix the problem by running \"debootstrap --arch=amd64 quantal %v\"", testLayerPath) + return } } @@ -33,14 +49,34 @@ func newTestDocker() (*Docker, error) { if err != nil { return nil, err } + + if layer, err := layerArchive(testLayerPath); err != nil { + panic(err) + } else { + _, err = docker.Store.Create(layer, nil, "docker-ut", "unit tests") + if err != nil { + panic(err) + } + } return docker, nil } +func GetTestImage(docker *Docker) *fs.Image { + imgs, err := docker.Store.Images() + if err != nil { + panic(err) + } else if len(imgs) < 1 { + panic("GASP") + } + return imgs[0] +} + func TestCreate(t *testing.T) { docker, err := newTestDocker() if err != nil { t.Fatal(err) } + defer nuke(docker) // Make sure we start we 0 containers if len(docker.List()) != 0 { @@ -50,7 +86,7 @@ func TestCreate(t *testing.T) { "test_create", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -94,11 +130,12 @@ func TestDestroy(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "test_destroy", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -142,11 +179,12 @@ func TestGet(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container1, err := docker.Create( "test1", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -158,7 +196,7 @@ func TestGet(t *testing.T) { "test2", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -170,7 +208,7 @@ func TestGet(t *testing.T) { "test3", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -198,16 +236,27 @@ func TestRestore(t *testing.T) { t.Fatal(err) } - // Create a container with one instance of docker docker1, err := NewFromDirectory(root) if err != nil { t.Fatal(err) } + defer nuke(docker1) + + if layer, err := layerArchive(testLayerPath); err != nil { + panic(err) + } else { + _, err = docker1.Store.Create(layer, nil, "docker-ut", "unit tests") + if err != nil { + panic(err) + } + } + + // Create a container with one instance of docker container1, err := docker1.Create( "restore_test", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker1), &Config{}, ) if err != nil { @@ -227,6 +276,7 @@ func TestRestore(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker2) if len(docker2.List()) != 1 { t.Errorf("Expected 1 container, %v found", len(docker2.List())) } diff --git a/dockerd/dockerweb.html b/dockerd/dockerweb.html old mode 100644 new mode 100755 diff --git a/fake/fake.go b/fake/fake.go index 8edbd88e6..598ff6a81 100644 --- a/fake/fake.go +++ b/fake/fake.go @@ -13,7 +13,7 @@ func FakeTar() (io.Reader, error) { content := []byte("Hello world!\n") buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - for _, name := range []string{"hello", "etc/postgres/postgres.conf", "etc/passwd", "var/log/postgres/postgres.conf"} { + for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { hdr := new(tar.Header) hdr.Size = int64(len(content)) hdr.Name = name diff --git a/filesystem.go b/filesystem.go deleted file mode 100644 index 26686b3a5..000000000 --- a/filesystem.go +++ /dev/null @@ -1,253 +0,0 @@ -package docker - -import ( - "errors" - "fmt" - "github.com/dotcloud/docker/image" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "syscall" - "time" -) - -type Filesystem struct { - RootFS string - RWPath string - // The layers to be mounted on top of each other via aufs. - // Layers are ordered top-to-bottom: the first layer in the list will be mounted on top of the others. - // In other words, THE BASE IMAGE SHOULD BE LAST! - Layers []string -} - -func (fs *Filesystem) createMountPoints() error { - if err := os.Mkdir(fs.RootFS, 0755); err != nil && !os.IsExist(err) { - return err - } - if err := os.Mkdir(fs.RWPath, 0755); err != nil && !os.IsExist(err) { - return err - } - return nil -} - -func (fs *Filesystem) Mount() error { - if fs.IsMounted() { - return errors.New("Mount: Filesystem already mounted") - } - if err := fs.createMountPoints(); err != nil { - return err - } - rwBranch := fmt.Sprintf("%v=rw", fs.RWPath) - roBranches := "" - for _, layer := range fs.Layers { - roBranches += fmt.Sprintf("%v=ro:", layer) - } - branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) - if err := mount("none", fs.RootFS, "aufs", 0, branches); err != nil { - return err - } - if !fs.IsMounted() { - return errors.New("Mount failed") - } - return nil -} - -func (fs *Filesystem) Umount() error { - if !fs.IsMounted() { - return errors.New("Umount: Filesystem not mounted") - } - if err := syscall.Unmount(fs.RootFS, 0); err != nil { - return err - } - if fs.IsMounted() { - return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", fs.RootFS) - } - // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint - // for some time. We'll just keep retrying until it succeeds. - for retries := 0; retries < 1000; retries++ { - err := os.Remove(fs.RootFS) - if err == nil { - // rm mntpoint succeeded - return nil - } - if os.IsNotExist(err) { - // mntpoint doesn't exist anymore. Success. - return nil - } - // fmt.Printf("(%v) Remove %v returned: %v\n", retries, fs.RootFS, err) - time.Sleep(10 * time.Millisecond) - } - return fmt.Errorf("Umount: Failed to umount %v", fs.RootFS) -} - -func (fs *Filesystem) IsMounted() bool { - mntpoint, err := os.Stat(fs.RootFS) - if err != nil { - if os.IsNotExist(err) { - return false - } - panic(err) - } - parent, err := os.Stat(filepath.Join(fs.RootFS, "..")) - if err != nil { - panic(err) - } - - mntpointSt := mntpoint.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) - return mntpointSt.Dev != parentSt.Dev -} - -// Tar returns the contents of the filesystem as an uncompressed tar stream -func (fs *Filesystem) Tar() (io.Reader, error) { - if err := fs.EnsureMounted(); err != nil { - return nil, err - } - return image.Tar(fs.RootFS, image.Uncompressed) -} - -func (fs *Filesystem) EnsureMounted() error { - if !fs.IsMounted() { - if err := fs.Mount(); err != nil { - return err - } - } - return nil -} - -type ChangeType int - -const ( - ChangeModify = iota - ChangeAdd - ChangeDelete -) - -type Change struct { - Path string - Kind ChangeType -} - -func (change *Change) String() string { - var kind string - switch change.Kind { - case ChangeModify: - kind = "C" - case ChangeAdd: - kind = "A" - case ChangeDelete: - kind = "D" - } - return fmt.Sprintf("%s %s", kind, change.Path) -} - -func (fs *Filesystem) Changes() ([]Change, error) { - var changes []Change - err := filepath.Walk(fs.RWPath, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - - // Rebase path - path, err = filepath.Rel(fs.RWPath, path) - if err != nil { - return err - } - path = filepath.Join("/", path) - - // Skip root - if path == "/" { - return nil - } - - // Skip AUFS metadata - if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { - return err - } - - change := Change{ - Path: path, - } - - // Find out what kind of modification happened - file := filepath.Base(path) - // If there is a whiteout, then the file was removed - if strings.HasPrefix(file, ".wh.") { - originalFile := strings.TrimLeft(file, ".wh.") - change.Path = filepath.Join(filepath.Dir(path), originalFile) - change.Kind = ChangeDelete - } else { - // Otherwise, the file was added - change.Kind = ChangeAdd - - // ...Unless it already existed in a top layer, in which case, it's a modification - for _, layer := range fs.Layers { - stat, err := os.Stat(filepath.Join(layer, path)) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - // The file existed in the top layer, so that's a modification - - // However, if it's a directory, maybe it wasn't actually modified. - // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar - if stat.IsDir() && f.IsDir() { - if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { - // Both directories are the same, don't record the change - return nil - } - } - change.Kind = ChangeModify - break - } - } - } - - // Record change - changes = append(changes, change) - return nil - }) - if err != nil { - return nil, err - } - return changes, nil -} - -// Reset removes all changes to the filesystem, reverting it to its initial state. -func (fs *Filesystem) Reset() error { - if err := os.RemoveAll(fs.RWPath); err != nil { - return err - } - // We removed the RW directory itself along with its content: let's re-create an empty one. - if err := fs.createMountPoints(); err != nil { - return err - } - return nil -} - -// Open opens the named file for reading. -func (fs *Filesystem) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { - if err := fs.EnsureMounted(); err != nil { - return nil, err - } - return os.OpenFile(filepath.Join(fs.RootFS, path), flag, perm) -} - -// ReadDir reads the directory named by dirname, relative to the Filesystem's root, -// and returns a list of sorted directory entries -func (fs *Filesystem) ReadDir(dirname string) ([]os.FileInfo, error) { - if err := fs.EnsureMounted(); err != nil { - return nil, err - } - return ioutil.ReadDir(filepath.Join(fs.RootFS, dirname)) -} - -func newFilesystem(rootfs string, rwpath string, layers []string) *Filesystem { - return &Filesystem{ - RootFS: rootfs, - RWPath: rwpath, - Layers: layers, - } -} diff --git a/filesystem_test.go b/filesystem_test.go deleted file mode 100644 index 5f24ae62a..000000000 --- a/filesystem_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package docker - -import ( - "bytes" - "io/ioutil" - "os" - "path" - "testing" -) - -func newTestFilesystem(t *testing.T, layers []string) (rootfs string, fs *Filesystem) { - rootfs, err := ioutil.TempDir("", "docker-test-root") - if err != nil { - t.Fatal(err) - } - rwpath, err := ioutil.TempDir("", "docker-test-rw") - if err != nil { - t.Fatal(err) - } - fs = newFilesystem(rootfs, rwpath, layers) - return -} - -func TestFilesystem(t *testing.T) { - _, filesystem := newTestFilesystem(t, []string{testLayerPath}) - if err := filesystem.Umount(); err == nil { - t.Errorf("Umount succeeded even though the filesystem was not mounted") - } - - if filesystem.IsMounted() { - t.Fatal("Filesystem should not be mounted") - } - - if err := filesystem.Mount(); err != nil { - t.Fatal(err) - } - - if !filesystem.IsMounted() { - t.Fatal("Filesystem should be mounted") - } - - if err := filesystem.Mount(); err == nil { - t.Errorf("Double mount succeeded") - } - - if !filesystem.IsMounted() { - t.Fatal("Filesystem should be mounted") - } - - if err := filesystem.Umount(); err != nil { - t.Fatal(err) - } - - if filesystem.IsMounted() { - t.Fatal("Filesystem should not be mounted") - } - - if err := filesystem.Umount(); err == nil { - t.Errorf("Umount succeeded even though the filesystem was already umounted") - } - - if filesystem.IsMounted() { - t.Fatal("Filesystem should not be mounted") - } -} - -func TestFilesystemMultiLayer(t *testing.T) { - // Create a fake layer - fakeLayer, err := ioutil.TempDir("", "docker-layer") - if err != nil { - t.Fatal(err) - } - data := []byte("hello world") - if err := ioutil.WriteFile(path.Join(fakeLayer, "test_file"), data, 0700); err != nil { - t.Fatal(err) - } - - // Create the layered filesystem and add our fake layer on top - rootfs, filesystem := newTestFilesystem(t, []string{testLayerPath, fakeLayer}) - - // Mount it - if err := filesystem.Mount(); err != nil { - t.Fatal(err) - } - defer func() { - if err := filesystem.Umount(); err != nil { - t.Fatal(err) - } - }() - - // Check to see whether we can access our fake layer - if _, err := os.Stat(path.Join(rootfs, "test_file")); err != nil { - t.Fatal(err) - } - fsdata, err := ioutil.ReadFile(path.Join(rootfs, "test_file")) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, fsdata) { - t.Error(string(fsdata)) - } -} - -func TestChanges(t *testing.T) { - rootfs, filesystem := newTestFilesystem(t, []string{testLayerPath}) - // Mount it - if err := filesystem.Mount(); err != nil { - t.Fatal(err) - } - defer filesystem.Umount() - - var changes []Change - var err error - - // Test without changes - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 0 { - t.Errorf("Unexpected changes :%v", changes) - } - - // Test simple change - file, err := os.Create(path.Join(rootfs, "test_change")) - if err != nil { - t.Fatal(err) - } - file.Close() - - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 1 { - t.Errorf("Unexpected changes :%v", changes) - } - if changes[0].Path != "/test_change" || changes[0].Kind != ChangeAdd { - t.Errorf("Unexpected changes :%v", changes) - } - - // Test subdirectory change - if err := os.Mkdir(path.Join(rootfs, "sub_change"), 0700); err != nil { - t.Fatal(err) - } - - file, err = os.Create(path.Join(rootfs, "sub_change", "test")) - if err != nil { - t.Fatal(err) - } - file.Close() - - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 3 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[0].Path != "/sub_change" || changes[0].Kind != ChangeAdd || changes[1].Path != "/sub_change/test" || changes[1].Kind != ChangeAdd { - t.Errorf("Unexpected changes: %v", changes) - } - - // Test permission change - if err := os.Chmod(path.Join(rootfs, "root"), 0000); err != nil { - t.Fatal(err) - } - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 4 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[0].Path != "/root" || changes[0].Kind != ChangeModify { - t.Errorf("Unexpected changes: %v", changes) - } - - // Test removal - if err := os.Remove(path.Join(rootfs, "etc", "passwd")); err != nil { - t.Fatal(err) - } - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 6 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[0].Path != "/etc" || changes[0].Kind != ChangeModify || changes[1].Path != "/etc/passwd" || changes[1].Kind != ChangeDelete { - t.Errorf("Unexpected changes: %v", changes) - } - - // Test sub-directory removal - if err := os.Remove(path.Join(rootfs, "usr", "bin", "sudo")); err != nil { - t.Fatal(err) - } - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 8 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[6].Path != "/usr/bin" || changes[6].Kind != ChangeModify || changes[7].Path != "/usr/bin/sudo" || changes[7].Kind != ChangeDelete { - t.Errorf("Unexpected changes: %v", changes) - } -} diff --git a/fs/archive.go b/fs/archive.go new file mode 100644 index 000000000..f43cc64b6 --- /dev/null +++ b/fs/archive.go @@ -0,0 +1,73 @@ +package fs + +import ( + "errors" + "io" + "io/ioutil" + "os/exec" +) + +type Compression uint32 + +const ( + Uncompressed Compression = iota + Bzip2 + Gzip +) + +func (compression *Compression) Flag() string { + switch *compression { + case Bzip2: + return "j" + case Gzip: + return "z" + } + return "" +} + +func Tar(path string, compression Compression) (io.Reader, error) { + cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".") + return CmdStream(cmd) +} + +func Untar(archive io.Reader, path string) error { + cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x") + cmd.Stdin = archive + output, err := cmd.CombinedOutput() + if err != nil { + return errors.New(err.Error() + ": " + string(output)) + } + return nil +} + +func CmdStream(cmd *exec.Cmd) (io.Reader, error) { + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + pipeR, pipeW := io.Pipe() + go func() { + _, err := io.Copy(pipeW, stdout) + if err != nil { + pipeW.CloseWithError(err) + } + errText, e := ioutil.ReadAll(stderr) + if e != nil { + errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")") + } + if err := cmd.Wait(); err != nil { + // FIXME: can this block if stderr outputs more than the size of StderrPipe()'s buffer? + pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText))) + } else { + pipeW.Close() + } + }() + if err := cmd.Start(); err != nil { + return nil, err + } + return pipeR, nil +} diff --git a/fs/archive_test.go b/fs/archive_test.go new file mode 100644 index 000000000..b182a1563 --- /dev/null +++ b/fs/archive_test.go @@ -0,0 +1,54 @@ +package fs + +import ( + "io/ioutil" + "os" + "os/exec" + "testing" +) + +func TestCmdStreamBad(t *testing.T) { + badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") + out, err := CmdStream(badCmd) + if err != nil { + t.Fatalf("Failed to start command: " + err.Error()) + } + if output, err := ioutil.ReadAll(out); err == nil { + t.Fatalf("Command should have failed") + } else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" { + t.Fatalf("Wrong error value (%s)", err.Error()) + } else if s := string(output); s != "hello\n" { + t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output) + } +} + +func TestCmdStreamGood(t *testing.T) { + cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0") + out, err := CmdStream(cmd) + if err != nil { + t.Fatal(err) + } + if output, err := ioutil.ReadAll(out); err != nil { + t.Fatalf("Command should not have failed (err=%s)", err) + } else if s := string(output); s != "hello\n" { + t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output) + } +} + +func TestTarUntar(t *testing.T) { + archive, err := Tar(".", Uncompressed) + if err != nil { + t.Fatal(err) + } + tmp, err := ioutil.TempDir("", "docker-test-untar") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + if err := Untar(archive, tmp); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(tmp); err != nil { + t.Fatalf("Error stating %s: %s", tmp, err.Error()) + } +} diff --git a/fs/changes.go b/fs/changes.go new file mode 100644 index 000000000..659f688c4 --- /dev/null +++ b/fs/changes.go @@ -0,0 +1,127 @@ +package fs + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type ChangeType int + +const ( + ChangeModify = iota + ChangeAdd + ChangeDelete +) + +type Change struct { + Path string + Kind ChangeType +} + +func (change *Change) String() string { + var kind string + switch change.Kind { + case ChangeModify: + kind = "C" + case ChangeAdd: + kind = "A" + case ChangeDelete: + kind = "D" + } + return fmt.Sprintf("%s %s", kind, change.Path) +} + +func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { + var changes []Change + image, err := store.Get(mp.Image) + if err != nil { + return nil, err + } + layers, err := image.layers() + if err != nil { + return nil, err + } + + err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(mp.Rw, path) + if err != nil { + return err + } + path = filepath.Join("/", path) + + // Skip root + if path == "/" { + return nil + } + + // Skip AUFS metadata + if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { + return err + } + + change := Change{ + Path: path, + } + + // Find out what kind of modification happened + file := filepath.Base(path) + // If there is a whiteout, then the file was removed + if strings.HasPrefix(file, ".wh.") { + originalFile := strings.TrimLeft(file, ".wh.") + change.Path = filepath.Join(filepath.Dir(path), originalFile) + change.Kind = ChangeDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeAdd + + // ...Unless it already existed in a top layer, in which case, it's a modification + for _, layer := range layers { + stat, err := os.Stat(filepath.Join(layer, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the top layer, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeModify + break + } + } + } + + // Record change + changes = append(changes, change) + return nil + }) + if err != nil { + return nil, err + } + return changes, nil +} + +// Reset removes all changes to the filesystem, reverting it to its initial state. +func (mp *Mountpoint) Reset() error { + if err := os.RemoveAll(mp.Rw); err != nil { + return err + } + // We removed the RW directory itself along with its content: let's re-create an empty one. + if err := mp.createFolders(); err != nil { + return err + } + return nil +} diff --git a/image/layers.go b/fs/layers.go similarity index 79% rename from image/layers.go rename to fs/layers.go index 7a24b7bd5..be699a8a0 100644 --- a/image/layers.go +++ b/fs/layers.go @@ -1,7 +1,8 @@ -package image +package fs import ( "errors" + "fmt" "github.com/dotcloud/docker/future" "io" "io/ioutil" @@ -19,6 +20,10 @@ func NewLayerStore(root string) (*LayerStore, error) { if err != nil { return nil, err } + // Create the root directory if it doesn't exists + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } return &LayerStore{ Root: abspath, }, nil @@ -79,35 +84,29 @@ func (store *LayerStore) layerPath(id string) string { return path.Join(store.Root, id) } -func (store *LayerStore) AddLayer(archive io.Reader) (string, error) { +func (store *LayerStore) AddLayer(id string, archive Archive) (string, error) { + if _, err := os.Stat(store.layerPath(id)); err == nil { + return "", fmt.Errorf("Layer already exists: %v", id) + } errors := make(chan error) // Untar tmp, err := store.Mktemp() defer os.RemoveAll(tmp) if err != nil { - return "", err + return "", fmt.Errorf("Mktemp failed: %s", err) } + untarR, untarW := io.Pipe() go func() { errors <- Untar(untarR, tmp) }() - // Compute ID - var id string - hashR, hashW := io.Pipe() - go func() { - _id, err := future.ComputeId(hashR) - id = _id - errors <- err - }() - // Duplicate archive to each stream - _, err = io.Copy(io.MultiWriter(hashW, untarW), archive) - hashW.Close() + _, err = io.Copy(untarW, archive) untarW.Close() if err != nil { return "", err } // Wait for goroutines - for i := 0; i < 2; i += 1 { + for i := 0; i < 1; i += 1 { select { case err := <-errors: { @@ -120,7 +119,7 @@ func (store *LayerStore) AddLayer(archive io.Reader) (string, error) { layer := store.layerPath(id) if !store.Exists(id) { if err := os.Rename(tmp, layer); err != nil { - return "", err + return "", fmt.Errorf("Could not rename temp dir to layer %s: %s", layer, err) } } return layer, nil diff --git a/fs/layers_test.go b/fs/layers_test.go new file mode 100644 index 000000000..9f8353fdd --- /dev/null +++ b/fs/layers_test.go @@ -0,0 +1,77 @@ +package fs + +import ( + "github.com/dotcloud/docker/fake" + "io/ioutil" + "os" + "testing" +) + +func TestLayersInit(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + // Root should exist + if _, err := os.Stat(store.Root); err != nil { + t.Fatal(err) + } + // List() should be empty + if l := store.List(); len(l) != 0 { + t.Fatalf("List() should return %d, not %d", 0, len(l)) + } +} + +func TestAddLayer(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + layer, err := store.AddLayer("foo", testArchive(t)) + if err != nil { + t.Fatal(err) + } + // Layer path should exist + if _, err := os.Stat(layer); err != nil { + t.Fatal(err) + } + // List() should return 1 layer + if l := store.List(); len(l) != 1 { + t.Fatalf("List() should return %d elements, not %d", 1, len(l)) + } + // Get("foo") should return the correct layer + if foo := store.Get("foo"); foo != layer { + t.Fatalf("get(\"foo\") should return '%d', not '%d'", layer, foo) + } +} + +func TestAddLayerDuplicate(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + if _, err := store.AddLayer("foobar123", testArchive(t)); err != nil { + t.Fatal(err) + } + if _, err := store.AddLayer("foobar123", testArchive(t)); err == nil { + t.Fatalf("Creating duplicate layer should fail") + } +} + +/* + * HELPER FUNCTIONS + */ + +func tempStore(t *testing.T) *LayerStore { + tmp, err := ioutil.TempDir("", "docker-fs-layerstore-") + if err != nil { + t.Fatal(err) + } + store, err := NewLayerStore(tmp) + if err != nil { + t.Fatal(err) + } + return store +} + +func testArchive(t *testing.T) Archive { + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + return archive +} diff --git a/fs/mount_darwin.go b/fs/mount_darwin.go new file mode 100644 index 000000000..540d6f769 --- /dev/null +++ b/fs/mount_darwin.go @@ -0,0 +1,7 @@ +package fs + +import "errors" + +func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { + return errors.New("mount is not implemented on darwin") +} diff --git a/fs/mount_linux.go b/fs/mount_linux.go new file mode 100644 index 000000000..b36888f75 --- /dev/null +++ b/fs/mount_linux.go @@ -0,0 +1,7 @@ +package fs + +import "syscall" + +func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { + return syscall.Mount(source, target, fstype, flags, data) +} diff --git a/fs/store.go b/fs/store.go new file mode 100644 index 000000000..36449a0fb --- /dev/null +++ b/fs/store.go @@ -0,0 +1,444 @@ +package fs + +import ( + "database/sql" + "errors" + "fmt" + "github.com/dotcloud/docker/future" + _ "github.com/mattn/go-sqlite3" + "github.com/shykes/gorp" //Forked to implement CreateTablesOpts + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "syscall" + "time" +) + +type Store struct { + Root string + db *sql.DB + orm *gorp.DbMap + layers *LayerStore +} + +type Archive io.Reader + +func New(root string) (*Store, error) { + isNewStore := true + + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } + db, err := sql.Open("sqlite3", path.Join(root, "db")) + if err != nil { + return nil, err + } + orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} + orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") + orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") + orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") + orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") + if isNewStore { + if err := orm.CreateTablesOpts(true); err != nil { + return nil, err + } + } + + layers, err := NewLayerStore(path.Join(root, "layers")) + if err != nil { + return nil, err + } + return &Store{ + Root: root, + db: db, + orm: orm, + layers: layers, + }, nil +} + +func (store *Store) imageList(src []interface{}) []*Image { + var images []*Image + for _, i := range src { + img := i.(*Image) + img.store = store + images = append(images, img) + } + return images +} + +func (store *Store) Images() ([]*Image, error) { + images, err := store.orm.Select(Image{}, "select * from images") + if err != nil { + return nil, err + } + return store.imageList(images), nil +} + +func (store *Store) Paths() ([]string, error) { + var paths []string + rows, err := store.db.Query("select distinct Path from paths order by Path") + if err != nil { + return nil, err + } + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, nil +} + +func (store *Store) List(pth string) ([]*Image, error) { + pth = path.Clean(pth) + images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id", pth) + if err != nil { + return nil, err + } + return store.imageList(images), nil +} + +func (store *Store) Find(pth string) (*Image, error) { + pth = path.Clean(pth) + img, err := store.Get(pth) + if err != nil { + return nil, err + } else if img != nil { + return img, nil + } + + images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc limit 1", pth) + if err != nil { + return nil, err + } else if len(images) < 1 { + return nil, nil + } + img = images[0].(*Image) + img.store = store + return img, nil +} + +func (store *Store) Get(id string) (*Image, error) { + img, err := store.orm.Get(Image{}, id) + if img == nil { + return nil, err + } + res := img.(*Image) + res.store = store + return res, err +} + +func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { + // FIXME: actually do something with the layer... + img := &Image{ + Id: future.RandomId(), + Comment: comment, + Created: time.Now().Unix(), + store: store, + } + if parent != nil { + img.Parent = parent.Id + } + // FIXME: we shouldn't have to pass os.Stderr to AddLayer()... + // FIXME: Archive should contain compression info. For now we only support uncompressed. + _, err := store.layers.AddLayer(img.Id, layerData) + if err != nil { + return nil, errors.New(fmt.Sprintf("Could not add layer: %s", err)) + } + path := &Path{ + Path: path.Clean(pth), + Image: img.Id, + } + trans, err := store.orm.Begin() + if err != nil { + return nil, errors.New(fmt.Sprintf("Could not begin transaction:", err)) + } + if err := trans.Insert(img); err != nil { + return nil, errors.New(fmt.Sprintf("Could not insert image info: %s", err)) + } + if err := trans.Insert(path); err != nil { + return nil, errors.New(fmt.Sprintf("Could not insert path info: %s", err)) + } + if err := trans.Commit(); err != nil { + return nil, errors.New(fmt.Sprintf("Could not commit transaction: %s", err)) + } + return img, nil +} + +func (store *Store) Register(image *Image, pth string) error { + image.store = store + // FIXME: import layer + trans, err := store.orm.Begin() + if err != nil { + return err + } + trans.Insert(image) + trans.Insert(&Path{Path: pth, Image: image.Id}) + return trans.Commit() +} + +func (store *Store) Layers() []string { + return store.layers.List() +} + +type Image struct { + Id string + Parent string + Comment string + Created int64 + store *Store `db:"-"` +} + +func (image *Image) Copy(pth string) (*Image, error) { + if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { + return nil, err + } + return image, nil +} + +type Mountpoint struct { + Image string + Root string + Rw string + Store *Store `db:"-"` +} + +func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { + mountpoint := &Mountpoint{ + Root: path.Clean(root), + Rw: path.Clean(rw), + Image: image.Id, + Store: image.store, + } + if err := image.store.orm.Insert(mountpoint); err != nil { + return nil, err + } + return mountpoint, nil +} + +func (image *Image) layers() ([]string, error) { + var list []string + var err error + currentImg := image + for currentImg != nil { + if layer := image.store.layers.Get(currentImg.Id); layer != "" { + list = append(list, layer) + } else { + return list, fmt.Errorf("Layer not found for image %s", image.Id) + } + currentImg, err = currentImg.store.Get(currentImg.Parent) + if err != nil { + return list, fmt.Errorf("Error while getting parent image: %v", err) + } + } + return list, nil +} + +func (image *Image) Mountpoints() ([]*Mountpoint, error) { + var mountpoints []*Mountpoint + res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) + if err != nil { + return nil, err + } + for _, mp := range res { + mountpoints = append(mountpoints, mp.(*Mountpoint)) + } + return mountpoints, nil +} + +func (image *Image) Mount(root, rw string) (*Mountpoint, error) { + var mountpoint *Mountpoint + if mp, err := image.store.FetchMountpoint(root, rw); err != nil { + return nil, err + } else if mp == nil { + mountpoint, err = image.Mountpoint(root, rw) + if err != nil { + return nil, fmt.Errorf("Could not create mountpoint: %s", err) + } else if mountpoint == nil { + return nil, errors.New("No mountpoint created") + } + } else { + mountpoint = mp + } + + if err := mountpoint.createFolders(); err != nil { + return nil, err + } + + // FIXME: Now mount the layers + rwBranch := fmt.Sprintf("%v=rw", mountpoint.Rw) + roBranches := "" + layers, err := image.layers() + if err != nil { + return nil, err + } + for _, layer := range layers { + roBranches += fmt.Sprintf("%v=ro:", layer) + } + branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) + if err := mount("none", mountpoint.Root, "aufs", 0, branches); err != nil { + return mountpoint, err + } + if !mountpoint.Mounted() { + return mountpoint, errors.New("Mount failed") + } + + return mountpoint, nil +} + +func (mp *Mountpoint) EnsureMounted() error { + if mp.Mounted() { + return nil + } + img, err := mp.Store.Get(mp.Image) + if err != nil { + return err + } + + _, err = img.Mount(mp.Root, mp.Rw) + return err +} + +func (mp *Mountpoint) createFolders() error { + if err := os.Mkdir(mp.Root, 0755); err != nil && !os.IsExist(err) { + return err + } + if err := os.Mkdir(mp.Rw, 0755); err != nil && !os.IsExist(err) { + return err + } + return nil +} + +func (mp *Mountpoint) Mounted() bool { + root, err := os.Stat(mp.Root) + if err != nil { + if os.IsNotExist(err) { + return false + } + panic(err) + } + parent, err := os.Stat(filepath.Join(mp.Root, "..")) + if err != nil { + panic(err) + } + + rootSt := root.Sys().(*syscall.Stat_t) + parentSt := parent.Sys().(*syscall.Stat_t) + return rootSt.Dev != parentSt.Dev +} + +func (mp *Mountpoint) Umount() error { + if !mp.Mounted() { + return errors.New("Mountpoint doesn't seem to be mounted") + } + if err := syscall.Unmount(mp.Root, 0); err != nil { + return fmt.Errorf("Unmount syscall failed: %v", err) + } + if mp.Mounted() { + return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", mp.Root) + } + // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint + // for some time. We'll just keep retrying until it succeeds. + for retries := 0; retries < 1000; retries++ { + err := os.Remove(mp.Root) + if err == nil { + // rm mntpoint succeeded + return nil + } + if os.IsNotExist(err) { + // mntpoint doesn't exist anymore. Success. + return nil + } + // fmt.Printf("(%v) Remove %v returned: %v\n", retries, mp.Root, err) + time.Sleep(10 * time.Millisecond) + } + return fmt.Errorf("Umount: Failed to umount %v", mp.Root) + +} + +func (mp *Mountpoint) Deregister() error { + if mp.Mounted() { + return errors.New("Mountpoint is currently mounted, can't deregister") + } + + _, err := mp.Store.orm.Delete(mp) + return err +} + +func (store *Store) FetchMountpoint(root, rw string) (*Mountpoint, error) { + res, err := store.orm.Select(Mountpoint{}, "select * from mountpoints where Root=? and Rw=?", root, rw) + if err != nil { + return nil, err + } else if len(res) < 1 || res[0] == nil { + return nil, nil + } + + mp := res[0].(*Mountpoint) + mp.Store = store + return mp, nil +} + +// OpenFile opens the named file for reading. +func (mp *Mountpoint) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { + if err := mp.EnsureMounted(); err != nil { + return nil, err + } + return os.OpenFile(filepath.Join(mp.Root, path), flag, perm) +} + +// ReadDir reads the directory named by dirname, relative to the Mountpoint's root, +// and returns a list of sorted directory entries +func (mp *Mountpoint) ReadDir(dirname string) ([]os.FileInfo, error) { + if err := mp.EnsureMounted(); err != nil { + return nil, err + } + return ioutil.ReadDir(filepath.Join(mp.Root, dirname)) +} + +func (store *Store) AddTag(imageId, tagName string) error { + if image, err := store.Get(imageId); err != nil { + return err + } else if image == nil { + return errors.New("No image with ID " + imageId) + } + + err2 := store.orm.Insert(&Tag{ + TagName: tagName, + Image: imageId, + }) + + return err2 +} + +func (store *Store) GetByTag(tagName string) (*Image, error) { + res, err := store.orm.Get(Tag{}, tagName) + if err != nil { + return nil, err + } else if res == nil { + return nil, errors.New("No image associated to tag \"" + tagName + "\"") + } + + tag := res.(*Tag) + + img, err2 := store.Get(tag.Image) + if err2 != nil { + return nil, err2 + } else if img == nil { + return nil, errors.New("Tag was found but image seems to be inexistent.") + } + + return img, nil +} + +type Path struct { + Path string + Image string +} + +type Tag struct { + TagName string + Image string +} diff --git a/fs/store_test.go b/fs/store_test.go new file mode 100644 index 000000000..f2cc06534 --- /dev/null +++ b/fs/store_test.go @@ -0,0 +1,298 @@ +package fs + +import ( + "errors" + "fmt" + "github.com/dotcloud/docker/fake" + "io/ioutil" + "os" + "testing" +) + +func TestInit(t *testing.T) { + store, err := TempStore("testinit") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + paths, err := store.Paths() + if err != nil { + t.Fatal(err) + } + if l := len(paths); l != 0 { + t.Fatal("Fresh store should be empty after init (len=%d)", l) + } +} + +func TestCreate(t *testing.T) { + store, err := TempStore("testcreate") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + if images, err := store.Images(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + if images, err := store.List("foo"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) + } else if images[0].Id != image.Id { + t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) + } +} + +func TestTag(t *testing.T) { + store, err := TempStore("testtag") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + if images, err := store.Images(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + + if err := store.AddTag(image.Id, "baz"); err != nil { + t.Fatalf("Error while adding a tag to created image: %s", err) + } + + if taggedImage, err := store.GetByTag("baz"); err != nil { + t.Fatalf("Error while trying to retrieve image for tag 'baz': %s", err) + } else if taggedImage.Id != image.Id { + t.Fatalf("Expected to retrieve image %s but found %s instead", image.Id, taggedImage.Id) + } +} + +// Copy an image to a new path +func TestCopyNewPath(t *testing.T) { + store, err := TempStore("testcopynewpath") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + src, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + dst, err := src.Copy("bar") + if err != nil { + t.Fatal(err) + } + // ID should be the same + if src.Id != dst.Id { + t.Fatal("Different IDs") + } + // Check number of images at source path + if images, err := store.List("foo"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatal("Wrong number of images at source path (should be %d, not %d)", 1, l) + } + // Check number of images at destination path + if images, err := store.List("bar"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatal("Wrong number of images at destination path (should be %d, not %d)", 1, l) + } + if err := healthCheck(store); err != nil { + t.Fatal(err) + } +} + +// Copying an image to the same path twice should fail +func TestCopySameName(t *testing.T) { + store, err := TempStore("testcopysamename") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + src, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + _, err = src.Copy("foo") + if err == nil { + t.Fatal("Copying an image to the same patch twice should fail.") + } +} + +func TestMountPoint(t *testing.T) { + store, err := TempStore("test-mountpoint") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + mountpoint, err := image.Mountpoint("/tmp/a", "/tmp/b") + if err != nil { + t.Fatal(err) + } + if mountpoint.Root != "/tmp/a" { + t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root) + } + if mountpoint.Rw != "/tmp/b" { + t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw) + } +} + +func TestMountpointDuplicateRoot(t *testing.T) { + store, err := TempStore("test-mountpoint") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + _, err = image.Mountpoint("/tmp/a", "/tmp/b") + if err != nil { + t.Fatal(err) + } + if _, err = image.Mountpoint("/tmp/a", "/tmp/foobar"); err == nil { + t.Fatal("Duplicate mountpoint root should fail") + } +} + +func TestMount(t *testing.T) { + store, err := TempStore("test-mount") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + // Create mount targets + root, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + rw, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + mountpoint, err := image.Mount(root, rw) + if err != nil { + t.Fatal(err) + } + defer mountpoint.Umount() + // Mountpoint should be marked as mounted + if !mountpoint.Mounted() { + t.Fatal("Mountpoint not mounted") + } + // There should be one mountpoint registered + if mps, err := image.Mountpoints(); err != nil { + t.Fatal(err) + } else if len(mps) != 1 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, len(mps)) + } + // Unmounting should work + if err := mountpoint.Umount(); err != nil { + t.Fatal(err) + } + // De-registering should work + if err := mountpoint.Deregister(); err != nil { + t.Fatal(err) + } + if mps, err := image.Mountpoints(); err != nil { + t.Fatal(err) + } else if len(mps) != 0 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, len(mps)) + } + // General health check + if err := healthCheck(store); err != nil { + t.Fatal(err) + } +} + +func TempStore(prefix string) (*Store, error) { + dir, err := ioutil.TempDir("", "docker-fs-test-"+prefix) + if err != nil { + return nil, err + } + return New(dir) +} + +func nuke(store *Store) error { + return os.RemoveAll(store.Root) +} + +// Look for inconsistencies in a store. +func healthCheck(store *Store) error { + parents := make(map[string]bool) + paths, err := store.Paths() + if err != nil { + return err + } + for _, path := range paths { + images, err := store.List(path) + if err != nil { + return err + } + IDs := make(map[string]bool) // All IDs for this path + for _, img := range images { + // Check for duplicate IDs per path + if _, exists := IDs[img.Id]; exists { + return errors.New(fmt.Sprintf("Duplicate ID: %s", img.Id)) + } else { + IDs[img.Id] = true + } + // Store parent for 2nd pass + if parent := img.Parent; parent != "" { + parents[parent] = true + } + } + } + // Check non-existing parents + for parent := range parents { + if _, exists := parents[parent]; !exists { + return errors.New("Reference to non-registered parent: " + parent) + } + } + return nil +} diff --git a/future/future.go b/future/future.go index 0063794e0..e7f3d9289 100644 --- a/future/future.go +++ b/future/future.go @@ -114,7 +114,7 @@ func (r *progressReader) Read(p []byte) (n int, err error) { // Only update progress for every 1% read update_every := int(0.01 * float64(r.read_total)) if r.read_progress - r.last_update > update_every { - fmt.Fprintf(r.output, "%d/%d (%.0f%%)\n", + fmt.Fprintf(r.output, "%d/%d (%.0f%%)\r", r.read_progress, r.read_total, float64(r.read_progress) / float64(r.read_total) * 100) diff --git a/image/image.go b/image/image.go deleted file mode 100644 index d9326fbd1..000000000 --- a/image/image.go +++ /dev/null @@ -1,368 +0,0 @@ -package image - -import ( - "encoding/json" - "errors" - "github.com/dotcloud/docker/future" - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - "sort" - "strings" - "time" -) - -type Store struct { - *Index - Root string - Layers *LayerStore -} - -func New(root string) (*Store, error) { - abspath, err := filepath.Abs(root) - if err != nil { - return nil, err - } - if err := os.MkdirAll(abspath, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - layers, err := NewLayerStore(path.Join(root, "layers")) - if err != nil { - return nil, err - } - if err := layers.Init(); err != nil { - return nil, err - } - return &Store{ - Root: abspath, - Index: NewIndex(path.Join(root, "index.json")), - Layers: layers, - }, nil -} - -// Import creates a new image from the contents of `archive` and registers it in the store as `name`. -// If `parent` is not nil, it will registered as the parent of the new image. -func (store *Store) Import(name string, archive io.Reader, parent *Image) (*Image, error) { - if strings.Contains(name, ":") { - return nil, errors.New("Invalid image name: " + name) - } - layer, err := store.Layers.AddLayer(archive) - if err != nil { - return nil, err - } - layers := []string{layer} - if parent != nil { - layers = append(layers, parent.Layers...) - } - var parentId string - if parent != nil { - parentId = parent.Id - } - return store.Create(name, parentId, layers...) -} - -func (store *Store) Create(name string, source string, layers ...string) (*Image, error) { - if strings.Contains(name, ":") { - return nil, errors.New("Invalid image name: " + name) - } - image, err := NewImage(name, layers, source) - if err != nil { - return nil, err - } - if err := store.Index.Add(name, image); err != nil { - return nil, err - } - return image, nil -} - -// Index - -type Index struct { - Path string - ByName map[string]*History - ById map[string]*Image -} - -func NewIndex(path string) *Index { - return &Index{ - Path: path, - ByName: make(map[string]*History), - ById: make(map[string]*Image), - } -} - -func (index *Index) Exists(id string) bool { - _, exists := index.ById[id] - return exists -} - -func (index *Index) Find(idOrName string) *Image { - // Load - if err := index.load(); err != nil { - return nil - } - // Lookup by ID - if image, exists := index.ById[idOrName]; exists { - return image - } - // Lookup by name - if history, exists := index.ByName[idOrName]; exists && history.Len() > 0 { - return (*history)[0] - } - return nil -} - -func (index *Index) Add(name string, image *Image) error { - // Load - if err := index.load(); err != nil { - return err - } - if _, exists := index.ByName[name]; !exists { - index.ByName[name] = new(History) - } else { - // If this image is already the latest version, don't add it. - if (*index.ByName[name])[0].Id == image.Id { - return nil - } - } - index.ByName[name].Add(image) - index.ById[image.Id] = image - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -func (index *Index) Copy(srcNameOrId, dstName string) (*Image, error) { - if srcNameOrId == "" || dstName == "" { - return nil, errors.New("Illegal image name") - } - // Load - if err := index.load(); err != nil { - return nil, err - } - src := index.Find(srcNameOrId) - if src == nil { - return nil, errors.New("No such image: " + srcNameOrId) - } - dst, err := NewImage(dstName, src.Layers, src.Id) - if err != nil { - return nil, err - } - if err := index.Add(dstName, dst); err != nil { - return nil, err - } - // Save - if err := index.save(); err != nil { - return nil, err - } - return dst, nil -} - -func (index *Index) Rename(oldName, newName string) error { - // Load - if err := index.load(); err != nil { - return err - } - if _, exists := index.ByName[oldName]; !exists { - return errors.New("Can't rename " + oldName + ": no such image.") - } - if _, exists := index.ByName[newName]; exists { - return errors.New("Can't rename to " + newName + ": name is already in use.") - } - index.ByName[newName] = index.ByName[oldName] - delete(index.ByName, oldName) - // Change the ID of all images, since they include the name - for _, image := range *index.ByName[newName] { - if id, err := generateImageId(newName, image.Layers); err != nil { - return err - } else { - oldId := image.Id - image.Id = id - index.ById[id] = image - delete(index.ById, oldId) - } - } - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -// Delete deletes all images with the name `name` -func (index *Index) Delete(name string) error { - // Load - if err := index.load(); err != nil { - return err - } - if _, exists := index.ByName[name]; !exists { - return errors.New("No such image: " + name) - } - // Remove from index lookup - for _, image := range *index.ByName[name] { - delete(index.ById, image.Id) - } - // Remove from name lookup - delete(index.ByName, name) - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -// DeleteMatch deletes all images whose name matches `pattern` -func (index *Index) DeleteMatch(pattern string) error { - // Load - if err := index.load(); err != nil { - return err - } - for name, history := range index.ByName { - if match, err := regexp.MatchString(pattern, name); err != nil { - return err - } else if match { - // Remove from index lookup - for _, image := range *history { - delete(index.ById, image.Id) - } - // Remove from name lookup - delete(index.ByName, name) - } - } - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -func (index *Index) Names() []string { - if err := index.load(); err != nil { - return []string{} - } - var names []string - for name := range index.ByName { - names = append(names, name) - } - sort.Strings(names) - return names -} - -func (index *Index) load() error { - jsonData, err := ioutil.ReadFile(index.Path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - path := index.Path - if err := json.Unmarshal(jsonData, index); err != nil { - return err - } - index.Path = path - return nil -} - -func (index *Index) save() error { - jsonData, err := json.Marshal(index) - if err != nil { - return err - } - if err := ioutil.WriteFile(index.Path, jsonData, 0600); err != nil { - return err - } - return nil -} - -// History wraps an array of images so they can be sorted by date (most recent first) - -type History []*Image - -func (history *History) Len() int { - return len(*history) -} - -func (history *History) Less(i, j int) bool { - images := *history - return images[j].Created.Before(images[i].Created) -} - -func (history *History) Swap(i, j int) { - images := *history - tmp := images[i] - images[i] = images[j] - images[j] = tmp -} - -func (history *History) Add(image *Image) { - *history = append(*history, image) - sort.Sort(history) -} - -func (history *History) Del(id string) { - for idx, image := range *history { - if image.Id == id { - *history = append((*history)[:idx], (*history)[idx+1:]...) - } - } -} - -type Image struct { - Id string // Globally unique identifier - Layers []string // Absolute paths - Created time.Time - Parent string -} - -func (image *Image) IdParts() (string, string) { - if len(image.Id) < 8 { - return "", image.Id - } - hash := image.Id[len(image.Id)-8 : len(image.Id)] - name := image.Id[:len(image.Id)-9] - return name, hash -} - -func (image *Image) IdIsFinal() bool { - return len(image.Layers) == 1 -} - -func generateImageId(name string, layers []string) (string, error) { - if len(layers) == 0 { - return "", errors.New("No layers provided.") - } - var hash string - if len(layers) == 1 { - hash = path.Base(layers[0]) - } else { - var ids string - for _, layer := range layers { - ids += path.Base(layer) - } - if h, err := future.ComputeId(strings.NewReader(ids)); err != nil { - return "", err - } else { - hash = h - } - } - return name + ":" + hash, nil -} - -func NewImage(name string, layers []string, parent string) (*Image, error) { - id, err := generateImageId(name, layers) - if err != nil { - return nil, err - } - return &Image{ - Id: id, - Layers: layers, - Created: time.Now(), - Parent: parent, - }, nil -} diff --git a/install.sh b/install.sh index af5c214d7..0cfba4ddc 100644 --- a/install.sh +++ b/install.sh @@ -1,20 +1,46 @@ -# This script is meant for quick & easy install via 'curl URL-OF-SCRIPPT | bash' -# Courtesy of Jeff Lindsay +#!/bin/sh +# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | sh' +# Original version by Jeff Lindsay +# Revamped by Jerome Petazzoni +# +# This script canonical location is http://get.docker.io/; to update it, run: +# s3cmd put -m text/x-shellscript -P install.sh s3://get.docker.io/index -cd /tmp +echo "Ensuring basic dependencies are installed..." +apt-get -qq update +apt-get -qq install lxc wget bsdtar -echo "Ensuring dependencies are installed..." -apt-get --yes install lxc wget bsdtar 2>&1 > /dev/null +echo "Looking in /proc/filesystems to see if we have AUFS support..." +if grep -q aufs /proc/filesystems +then + echo "Found." +else + echo "Ahem, it looks like the current kernel does not support AUFS." + echo "Let's see if we can load the AUFS module with modprobe..." + if modprobe aufs + then + echo "Module loaded." + else + echo "Ahem, things didn't turn out as expected." + KPKG=linux-image-extra-$(uname -r) + echo "Trying to install $KPKG..." + if apt-get -qq install $KPKG + then + echo "Installed." + else + echo "Oops, we couldn't install the -extra kernel." + echo "Are you sure you are running a supported version of Ubuntu?" + echo "Proceeding anyway, but Docker will probably NOT WORK!" + fi + fi +fi -echo "Downloading docker binary..." -wget -q https://dl.dropbox.com/u/20637798/docker.tar.gz 2>&1 > /dev/null -tar -xf docker.tar.gz 2>&1 > /dev/null +echo "Downloading docker binary and uncompressing into /usr/local/bin..." +curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | +tar -C /usr/local/bin --strip-components=1 -zxf- \ +docker-master/docker docker-master/dockerd -echo "Installing into /usr/local/bin..." -mv docker/docker /usr/local/bin -mv dockerd/dockerd /usr/local/bin - -if [[ -f /etc/init/dockerd.conf ]] +if [ -f /etc/init/dockerd.conf ] then echo "Upstart script already exists." else @@ -22,13 +48,8 @@ else echo "exec /usr/local/bin/dockerd" > /etc/init/dockerd.conf fi -echo "Restarting dockerd..." -restart dockerd > /dev/null +echo "Starting dockerd..." +start dockerd > /dev/null -echo "Cleaning up..." -rmdir docker -rmdir dockerd -rm docker.tar.gz - -echo "Finished!" +echo "Done." echo diff --git a/lxc_template.go b/lxc_template.go index 931095c99..7ae50bce8 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -22,7 +22,7 @@ lxc.network.mtu = 1500 lxc.network.ipv4 = {{.NetworkSettings.IpAddress}}/{{.NetworkSettings.IpPrefixLen}} # root filesystem -{{$ROOTFS := .Filesystem.RootFS}} +{{$ROOTFS := .Mountpoint.Root}} lxc.rootfs = {{$ROOTFS}} # use a dedicated pts for the container (and limit the number of pseudo terminal @@ -85,16 +85,32 @@ lxc.mount.entry = /etc/resolv.conf {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config # limits -{{if .Config.Ram}} -lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} +{{if .Config.Memory}} +lxc.cgroup.memory.limit_in_bytes = {{.Config.Memory}} +lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Memory}} +{{with $memSwap := getMemorySwap .Config}} +lxc.cgroup.memory.memsw.limit_in_bytes = {{$memSwap}} +{{end}} {{end}} ` var LxcTemplateCompiled *template.Template +func getMemorySwap(config *Config) int64 { + // By default, MemorySwap is set to twice the size of RAM. + // If you want to omit MemorySwap, set it to `-1'. + if config.MemorySwap < 0 { + return 0 + } + return config.Memory * 2 +} + func init() { var err error - LxcTemplateCompiled, err = template.New("lxc").Parse(LxcTemplate) + funcMap := template.FuncMap{ + "getMemorySwap": getMemorySwap, + } + LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) if err != nil { panic(err) } diff --git a/puppet/modules/docker/manifests/init.pp b/puppet/modules/docker/manifests/init.pp index 2923b302f..38b40a8e3 100644 --- a/puppet/modules/docker/manifests/init.pp +++ b/puppet/modules/docker/manifests/init.pp @@ -1,14 +1,19 @@ class docker { # update this with latest docker binary distro - $docker_url = "https://dl.dropbox.com/u/20637798/docker.tar.gz" + $docker_url = "http://docker.io.s3.amazonaws.com/builds/$kernel/$hardwaremodel/docker-master.tgz" # update this with latest go binary distry $go_url = "http://go.googlecode.com/files/go1.0.3.linux-amd64.tar.gz" - Package { ensure => "installed" } - package { ["lxc", "debootstrap", "wget", "bsdtar"]: } + package { ["lxc", "debootstrap", "wget", "bsdtar", "git", + "linux-image-3.5.0-25-generic", + "linux-image-extra-3.5.0-25-generic", + "virtualbox-guest-utils", + "linux-headers-3.5.0-25-generic"]: } + + notify { "docker_url = $docker_url": withpath => true } exec { "debootstrap" : require => Package["debootstrap"], @@ -26,7 +31,7 @@ class docker { exec { "fetch-docker" : require => Package["wget"], command => "/usr/bin/wget -O - $docker_url | /bin/tar xz -C /home/vagrant", - creates => "/home/vagrant/docker/dockerd" + creates => "/home/vagrant/docker-master" } file { "/etc/init/dockerd.conf": @@ -39,10 +44,21 @@ class docker { exec { "copy-docker-bin" : require => Exec["fetch-docker"], - command => "/bin/cp /home/vagrant/docker/docker /usr/local/bin", + command => "/bin/cp /home/vagrant/docker-master/docker /usr/local/bin", creates => "/usr/local/bin/docker" } + exec { "copy-dockerd-bin" : + require => Exec["fetch-docker"], + command => "/bin/cp /home/vagrant/docker-master/dockerd /usr/local/bin", + creates => "/usr/local/bin/dockerd" + } + + exec { "vbox-add" : + require => Package["linux-headers-3.5.0-25-generic"], + command => "/etc/init.d/vboxadd setup", + } + service { "dockerd" : ensure => "running", start => "/sbin/initctl start dockerd", diff --git a/puppet/modules/docker/templates/dockerd.conf b/puppet/modules/docker/templates/dockerd.conf index a00d322bc..c05f60639 100644 --- a/puppet/modules/docker/templates/dockerd.conf +++ b/puppet/modules/docker/templates/dockerd.conf @@ -7,5 +7,6 @@ start on runlevel [3] respawn script - /home/vagrant/dockerd/dockerd + test -f /etc/default/locale && . /etc/default/locale || true + LANG=$LANG LC_ALL=$LANG /usr/local/bin/dockerd end script diff --git a/rcli/http.go b/rcli/http.go index e6cb5657d..cc8d3b149 100644 --- a/rcli/http.go +++ b/rcli/http.go @@ -1,13 +1,12 @@ package rcli import ( + "fmt" "net/http" "net/url" "path" - "fmt" ) - // Use this key to encode an RPC call into an URL, // eg. domain.tld/path/to/method?q=get_user&q=gordon const ARG_URL_KEY = "q" @@ -16,18 +15,16 @@ func URLToCall(u *url.URL) (method string, args []string) { return path.Base(u.Path), u.Query()[ARG_URL_KEY] } - func ListenAndServeHTTP(addr string, service Service) error { return http.ListenAndServe(addr, http.HandlerFunc( - func (w http.ResponseWriter, r *http.Request) { + func(w http.ResponseWriter, r *http.Request) { cmd, args := URLToCall(r.URL) if err := call(service, r.Body, &AutoFlush{w}, append([]string{cmd}, args...)...); err != nil { - fmt.Fprintf(w, "Error: " + err.Error() + "\n") + fmt.Fprintf(w, "Error: "+err.Error()+"\n") } })) } - type AutoFlush struct { http.ResponseWriter } diff --git a/rcli/tcp.go b/rcli/tcp.go index 0a06d459c..869a3bcdb 100644 --- a/rcli/tcp.go +++ b/rcli/tcp.go @@ -1,13 +1,13 @@ package rcli import ( + "bufio" + "encoding/json" + "fmt" "io" "io/ioutil" - "net" "log" - "fmt" - "encoding/json" - "bufio" + "net" ) // Connect to a remote endpoint using protocol `proto` and address `addr`, @@ -44,7 +44,7 @@ func ListenAndServe(proto, addr string, service Service) error { go func() { if err := Serve(conn, service); err != nil { log.Printf("Error: " + err.Error() + "\n") - fmt.Fprintf(conn, "Error: " + err.Error() + "\n") + fmt.Fprintf(conn, "Error: "+err.Error()+"\n") } conn.Close() }() @@ -53,7 +53,6 @@ func ListenAndServe(proto, addr string, service Service) error { return nil } - // Parse an rcli call on a new connection, and pass it to `service` if it // is valid. func Serve(conn io.ReadWriter, service Service) error { @@ -68,4 +67,3 @@ func Serve(conn io.ReadWriter, service Service) error { } return nil } - diff --git a/rcli/types.go b/rcli/types.go index b8572cd89..52079291b 100644 --- a/rcli/types.go +++ b/rcli/types.go @@ -8,13 +8,13 @@ package rcli // are the usual suspects. import ( + "errors" + "flag" "fmt" "io" - "reflect" - "flag" "log" + "reflect" "strings" - "errors" ) type Service interface { @@ -25,7 +25,6 @@ type Service interface { type Cmd func(io.ReadCloser, io.Writer, ...string) error type CmdMethod func(Service, io.ReadCloser, io.Writer, ...string) error - func call(service Service, stdin io.ReadCloser, stdout io.Writer, args ...string) error { if len(args) == 0 { args = []string{"help"} @@ -63,7 +62,7 @@ func getMethod(service Service, name string) Cmd { return nil } } - methodName := "Cmd"+strings.ToUpper(name[:1])+strings.ToLower(name[1:]) + methodName := "Cmd" + strings.ToUpper(name[:1]) + strings.ToLower(name[1:]) method, exists := reflect.TypeOf(service).MethodByName(methodName) if !exists { return nil @@ -91,4 +90,3 @@ func Subcmd(output io.Writer, name, signature, description string) *flag.FlagSet } return flags } - diff --git a/server/server.go b/server/server.go index 9c07d4221..9c8fee262 100644 --- a/server/server.go +++ b/server/server.go @@ -7,12 +7,13 @@ import ( "errors" "fmt" "github.com/dotcloud/docker" + "github.com/dotcloud/docker/fs" "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/rcli" "io" "io/ioutil" "net/url" + "net/http" "os" "path" "strconv" @@ -37,29 +38,42 @@ func (srv *Server) Name() string { return "docker" } +// FIXME: Stop violating DRY by repeating usage here and in Subcmd declarations func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]interface{}{ {"run", "Run a command in a container"}, {"ps", "Display a list of containers"}, - {"pull", "Download a tarball and create a container from it"}, - {"put", "Upload a tarball and create a container from it"}, - {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, - {"rm", "Remove containers"}, - {"kill", "Kill a running container"}, - {"wait", "Wait for the state of a container to change"}, - {"stop", "Stop a running container"}, - {"start", "Start a stopped container"}, - {"restart", "Restart a running container"}, - {"logs", "Fetch the logs of a container"}, + {"import", "Create a new filesystem image from the contents of a tarball"}, + {"attach", "Attach to a running container"}, + {"cat", "Write the contents of a container's file to standard output"}, + {"commit", "Create a new image from a container's changes"}, + {"cp", "Create a copy of IMAGE and call it NAME"}, + {"debug", "(debug only) (No documentation available)"}, {"diff", "Inspect changes on a container's filesystem"}, - {"commit", "Save the state of a container"}, - {"attach", "Attach to the standard inputs and outputs of a running container"}, - {"wait", "Block until a container exits, then print its exit code"}, - {"info", "Display system-wide information"}, - {"tar", "Stream the contents of a container as a tar archive"}, - {"web", "Generate a web UI"}, {"images", "List images"}, + {"info", "Display system-wide information"}, + {"inspect", "Return low-level information on a container"}, + {"kill", "Kill a running container"}, + {"layers", "(debug only) List filesystem layers"}, + {"logs", "Fetch the logs of a container"}, + {"ls", "List the contents of a container's directory"}, + {"mirror", "(debug only) (No documentation available)"}, + {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, + {"ps", "List containers"}, + {"reset", "Reset changes to a container's filesystem"}, + {"restart", "Restart a running container"}, + {"rm", "Remove a container"}, + {"rmimage", "Remove an image"}, + {"run", "Run a command in a new container"}, + {"start", "Start a stopped container"}, + {"stop", "Stop a running container"}, + {"tar", "Stream the contents of a container as a tar archive"}, + {"umount", "(debug only) Mount a container's filesystem"}, + {"version", "Show the docker version information"}, + {"wait", "Block until a container stops, then print its exit code"}, + {"web", "A web UI for docker"}, + {"write", "Write the contents of standard input to a container's file"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd...) } @@ -70,7 +84,6 @@ func (srv *Server) Help() string { func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] NAME", "Block until a container stops, then print its exit code.") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -87,19 +100,39 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } +// 'docker version': show version information +func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + fmt.Fprintf(stdout, "Version:%s\n", VERSION) + return nil +} + // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + images, _ := srv.images.Images() + var imgcount int + if images == nil { + imgcount = 0 + } else { + imgcount = len(images) + } + cmd := rcli.Subcmd(stdout, "info", "", "Display system-wide information.") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() > 0 { + cmd.Usage() + return nil + } fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, - len(srv.images.ById)) + imgcount) return nil } func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] NAME", "Stop a running container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -122,7 +155,6 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "restart", "[OPTIONS] NAME", "Restart a running container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -145,7 +177,6 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "start", "[OPTIONS] NAME", "Start a stopped container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -168,7 +199,6 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "umount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -177,7 +207,7 @@ func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...stri } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Umount(); err != nil { + if err := container.Mountpoint.Umount(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -191,7 +221,6 @@ func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...stri func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "mount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -200,7 +229,7 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Mount(); err != nil { + if err := container.Mountpoint.EnsureMounted(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -214,7 +243,6 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -223,7 +251,7 @@ func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) } name, path := cmd.Arg(0), cmd.Arg(1) if container := srv.containers.Get(name); container != nil { - if f, err := container.Filesystem.OpenFile(path, os.O_RDONLY, 0); err != nil { + if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { return err } else if _, err := io.Copy(stdout, f); err != nil { return err @@ -236,7 +264,6 @@ func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -245,7 +272,7 @@ func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...strin } name, path := cmd.Arg(0), cmd.Arg(1) if container := srv.containers.Get(name); container != nil { - if f, err := container.Filesystem.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { + if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { return err } else if _, err := io.Copy(f, stdin); err != nil { return err @@ -258,7 +285,6 @@ func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -267,7 +293,7 @@ func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) } name, path := cmd.Arg(0), cmd.Arg(1) if container := srv.containers.Get(name); container != nil { - if files, err := container.Filesystem.ReadDir(path); err != nil { + if files, err := container.Mountpoint.ReadDir(path); err != nil { return err } else { for _, f := range files { @@ -282,7 +308,6 @@ func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -293,8 +318,8 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.containers.Get(name); container != nil { obj = container - } else if image := srv.images.Find(name); image != nil { - obj = image + //} else if image, err := srv.images.List(name); image != nil { + // obj = image } else { return errors.New("No such container or image: " + name) } @@ -315,7 +340,6 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "port", "[OPTIONS] CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() != 2 { @@ -337,34 +361,34 @@ func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string } // 'docker rmi NAME' removes all images with the name NAME -func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") - fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - var err error - if *fl_regexp { - err = srv.images.DeleteMatch(name) - } else { - image := srv.images.Find(name) - if image == nil { - return errors.New("No such image: " + name) - } - err = srv.images.Delete(name) - } - if err != nil { - return err - } - } - return nil -} +// func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") +// fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 1 { +// cmd.Usage() +// return nil +// } +// for _, name := range cmd.Args() { +// var err error +// if *fl_regexp { +// err = srv.images.DeleteMatch(name) +// } else { +// image := srv.images.Find(name) +// if image == nil { +// return errors.New("No such image: " + name) +// } +// err = srv.images.Delete(name) +// } +// if err != nil { +// return err +// } +// } +// return nil +// } func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER", "Remove a container") @@ -401,48 +425,12 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } -func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] NAME", "Download a new image from a remote location") - if err := cmd.Parse(args); err != nil { - return nil - } - name := cmd.Arg(0) - if name == "" { - return errors.New("Not enough arguments") - } - if strings.Contains(name, ":") { - return errors.New("Invalid image name: " + name) - } - u, err := url.Parse(name) - if err != nil { - return err - } - if u.Scheme == "" { - u.Scheme = "http" - } - // FIXME: hardcode a mirror URL that does not depend on a single provider. - if u.Host == "" { - u.Host = "s3.amazonaws.com" - u.Path = path.Join("/docker.io/images", u.Path) - } - fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) - resp, err := future.Download(u.String(), stdout) - // FIXME: Validate ContentLength - archive := future.ProgressReader(resp.Body, int(resp.ContentLength), stdout) - if err != nil { - return err - } - fmt.Fprintf(stdout, "Downloading and unpacking to %s\n", name) - img, err := srv.images.Import(name, archive, nil) - if err != nil { - return err - } - fmt.Fprintln(stdout, img.Id) - return nil -} +func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] NAME", "Create a new filesystem image from the contents of a tarball") + fl_stdin := cmd.Bool("stdin", false, "Read tarball from stdin") + var archive io.Reader + var resp *http.Response -func (srv *Server) CmdPut(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "put", "[OPTIONS] NAME", "Import a new image from a local archive.") if err := cmd.Parse(args); err != nil { return nil } @@ -450,7 +438,32 @@ func (srv *Server) CmdPut(stdin io.ReadCloser, stdout io.Writer, args ...string) if name == "" { return errors.New("Not enough arguments") } - img, err := srv.images.Import(name, stdin, nil) + if *fl_stdin { + archive = stdin + } else { + u, err := url.Parse(name) + if err != nil { + return err + } + if u.Scheme == "" { + u.Scheme = "http" + } + // FIXME: hardcode a mirror URL that does not depend on a single provider. + if u.Host == "" { + u.Host = "s3.amazonaws.com" + u.Path = path.Join("/docker.io/images", u.Path) + } + fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) + // Download with curl (pretty progress bar) + // If curl is not available, fallback to http.Get() + resp, err = future.Download(u.String(), stdout) + if err != nil { + return err + } + archive = future.ProgressReader(resp.Body, int(resp.ContentLength), stdout) + } + fmt.Fprintf(stdout, "Unpacking to %s\n", name) + img, err := srv.images.Create(archive, nil, name, "") if err != nil { return err } @@ -462,7 +475,9 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") - cmd.Parse(args) + if err := cmd.Parse(args); err != nil { + return nil + } if cmd.NArg() > 1 { cmd.Usage() return nil @@ -475,23 +490,27 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri if !*quiet { fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") } - for _, name := range srv.images.Names() { + paths, err := srv.images.Paths() + if err != nil { + return err + } + for _, name := range paths { if nameFilter != "" && nameFilter != name { continue } - for idx, img := range *srv.images.ByName[name] { + ids, err := srv.images.List(name) + if err != nil { + return err + } + for idx, img := range ids { if *limit > 0 && idx >= *limit { break } if !*quiet { - id := img.Id - if !img.IdIsFinal() { - id += "..." - } for idx, field := range []string{ /* NAME */ name, - /* ID */ id, - /* CREATED */ future.HumanDuration(time.Now().Sub(img.Created)) + " ago", + /* ID */ img.Id, + /* CREATED */ future.HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", /* PARENT */ img.Parent, } { if idx == 0 { @@ -568,7 +587,7 @@ func (srv *Server) CmdLayers(stdin io.ReadCloser, stdout io.Writer, args ...stri if err := cmd.Parse(args); err != nil { return nil } - for _, layer := range srv.images.Layers.List() { + for _, layer := range srv.images.Layers() { fmt.Fprintln(stdout, layer) } return nil @@ -581,10 +600,16 @@ func (srv *Server) CmdCp(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := cmd.Parse(args); err != nil { return nil } - if newImage, err := srv.images.Copy(cmd.Arg(0), cmd.Arg(1)); err != nil { + if image, err := srv.images.Get(cmd.Arg(0)); err != nil { return err + } else if image == nil { + return errors.New("Image " + cmd.Arg(0) + " does not exist") } else { - fmt.Fprintln(stdout, newImage.Id) + if img, err := image.Copy(cmd.Arg(1)); err != nil { + return err + } else { + fmt.Fprintln(stdout, img.Id) + } } return nil } @@ -603,16 +628,21 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } if container := srv.containers.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := image.Tar(container.Filesystem.RWPath, image.Uncompressed) + rwTar, err := fs.Tar(container.Mountpoint.Rw, fs.Uncompressed) if err != nil { return err } // Create a new image from the container's base layers + a new layer from container changes - parentImg := srv.images.Find(container.GetUserData("image")) - img, err := srv.images.Import(imgName, rwTar, parentImg) + parentImg, err := srv.images.Get(container.Image) if err != nil { return err } + + img, err := srv.images.Create(rwTar, parentImg, imgName, "") + if err != nil { + return err + } + fmt.Fprintln(stdout, img.Id) return nil } @@ -632,7 +662,10 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) } name := cmd.Arg(0) if container := srv.containers.Get(name); container != nil { - data, err := container.Filesystem.Tar() + if err := container.Mountpoint.EnsureMounted(); err != nil { + return err + } + data, err := fs.Tar(container.Mountpoint.Root, fs.Uncompressed) if err != nil { return err } @@ -658,7 +691,7 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string if container := srv.containers.Get(cmd.Arg(0)); container == nil { return errors.New("No such container") } else { - changes, err := container.Filesystem.Changes() + changes, err := srv.images.Changes(container.Mountpoint) if err != nil { return err } @@ -681,7 +714,7 @@ func (srv *Server) CmdReset(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Reset(); err != nil { + if err := container.Mountpoint.Reset(); err != nil { return errors.New("Reset " + container.Id + ": " + err.Error()) } } @@ -711,10 +744,17 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *image.Image, ports []int, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *fs.Image, ports []int, user string, tty bool, openStdin bool, memory int64, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] - container, err := srv.containers.Create(id, cmd, args, img.Layers, - &docker.Config{Hostname: id, Ports: ports, User: user, Tty: tty, OpenStdin: openStdin}) + container, err := srv.containers.Create(id, cmd, args, img, + &docker.Config{ + Hostname: id, + Ports: ports, + User: user, + Tty: tty, + OpenStdin: openStdin, + Memory: memory, + }) if err != nil { return nil, err } @@ -798,8 +838,8 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") fl_comment := cmd.String("c", "", "Comment") + fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") var fl_ports ports - var img *image.Image cmd.Var(&fl_ports, "p", "Map a network port to the container") if err := cmd.Parse(args); err != nil { @@ -818,15 +858,6 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) name = "base" } - // Separate the name:version tag - if strings.Contains(name, ":") { - parts := strings.SplitN(name, ":", 2) - img_name = parts[0] - //img_version = parts[1] // Only here for reference - } else { - img_name = name - } - // Choose a default command if needed if len(cmdline) == 0 { *fl_stdin = true @@ -836,20 +867,32 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) } // Find the image - img = srv.images.Find(name) - if img == nil { + img, err := srv.images.Find(name) + if err != nil { + return err + } else if img == nil { + // Separate the name:version tag + if strings.Contains(name, ":") { + parts := strings.SplitN(name, ":", 2) + img_name = parts[0] + //img_version = parts[1] // Only here for reference + } else { + img_name = name + } + stdin_noclose := ioutil.NopCloser(stdin) - if err := srv.CmdPull(stdin_noclose, stdout, img_name); err != nil { + if err := srv.CmdImport(stdin_noclose, stdout, img_name); err != nil { return err } - img = srv.images.Find(name) - if img == nil { + img, err = srv.images.Find(name) + if err != nil || img == nil { return errors.New("Could not find image after downloading: " + name) } } // Create new container - container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, *fl_stdin, *fl_comment, cmdline[0], cmdline[1:]...) + container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, + *fl_stdin, *fl_memory, *fl_comment, cmdline[0], cmdline[1:]...) if err != nil { return errors.New("Error creating container: " + err.Error()) } @@ -907,16 +950,15 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) func New() (*Server, error) { future.Seed() - images, err := image.New("/var/lib/docker/images") - if err != nil { - return nil, err - } + // if err != nil { + // return nil, err + // } containers, err := docker.New() if err != nil { return nil, err } srv := &Server{ - images: images, + images: containers.Store, containers: containers, } return srv, nil @@ -963,5 +1005,5 @@ func (srv *Server) CmdWeb(stdin io.ReadCloser, stdout io.Writer, args ...string) type Server struct { containers *docker.Docker - images *image.Store + images *fs.Store }