From 902d660d2ff48727386718440162ed2a8399eaf0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 18:46:46 -0800 Subject: [PATCH 01/50] docker/fs: a robust filesytem store --- fs/store.go | 156 ++++++++++++++++++++++++++++++++++ fs/store_test.go | 217 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 fs/store.go create mode 100644 fs/store_test.go diff --git a/fs/store.go b/fs/store.go new file mode 100644 index 000000000..e42959762 --- /dev/null +++ b/fs/store.go @@ -0,0 +1,156 @@ +package fs + +import ( + "database/sql" + _ "github.com/mattn/go-sqlite3" + "github.com/coopernurse/gorp" + "os" + "io" + "path" + "github.com/dotcloud/docker/future" +) + +type Store struct { + Root string + db *sql.DB + orm *gorp.DbMap +} + +type Archive io.Reader + +func New(root string) (*Store, error) { + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } + db, err := sql.Open("sqlite3", path.Join(root, "db")) + if err != nil { + return nil, err + } + orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} + orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") + orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") + if err := orm.CreateTables(); err != nil { + return nil, err + } + return &Store{ + Root: root, + db: db, + orm: orm, + }, nil +} + +func (store *Store) imageList(src []interface{}) ([]*Image) { + var images []*Image + for _, i := range src { + img := i.(*Image) + img.store = store + images = append(images, img) + } + return images +} + +func (store *Store) Images() ([]*Image, error) { + images , err := store.orm.Select(Image{}, "select * from images") + if err != nil { + return nil, err + } + return store.imageList(images), nil +} + +func (store *Store) Paths() ([]string, error) { + var paths []string + rows, err := store.db.Query("select distinct Path from paths order by Path") + if err != nil { + return nil, err + } + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, nil +} + +func (store *Store) List(pth string) ([]*Image, error) { + pth = path.Clean(pth) + images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id", pth) + if err != nil { + return nil, err + } + return store.imageList(images), nil +} + +func (store *Store) Get(id string) (*Image, error) { + images, err := store.orm.Select(Image{}, "select * from images where Id=?", id) + if err != nil { + return nil, err + } + if len(images) < 1 { + return nil, os.ErrNotExist + } + return images[0].(*Image), nil +} + +func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (*Image, error) { + // FIXME: actually do something with the layer... + img := &Image{ + Id : future.RandomId(), + Comment: comment, + store: store, + } + path := &Path{ + Path: path.Clean(pth), + Image: img.Id, + } + trans, err := store.orm.Begin() + if err != nil { + return nil, err + } + if err := trans.Insert(img); err != nil { + return nil, err + } + if err := trans.Insert(path); err != nil { + return nil, err + } + if err := trans.Commit(); err != nil { + return nil, err + } + return img, nil +} + +func (store *Store) Register(image *Image, pth string) error { + // FIXME: import layer + trans, err := store.orm.Begin() + if err != nil { + return err + } + trans.Insert(image) + trans.Insert(&Path{Path: pth, Image: image.Id}) + return trans.Commit() +} + + + + +type Image struct { + Id string + Parent string + Comment string + store *Store `db:"-"` +} + + +func (image *Image) Copy(pth string) (*Image, error) { + if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { + return nil, err + } + return image, nil +} + + +type Path struct { + Path string + Image string +} diff --git a/fs/store_test.go b/fs/store_test.go new file mode 100644 index 000000000..940396227 --- /dev/null +++ b/fs/store_test.go @@ -0,0 +1,217 @@ +package fs + +import ( + "testing" + "io/ioutil" + "github.com/dotcloud/docker/fake" + "os" + "errors" + "fmt" +) + +func TestInit(t *testing.T) { + store, err := TempStore("testinit") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + paths, err := store.Paths() + if err != nil { + t.Fatal(err) + } + if l := len(paths); l != 0 { + t.Fatal("Fresh store should be empty after init (len=%d)", l) + } +} + +func TestCreate(t *testing.T) { + store, err := TempStore("testcreate") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + if images, err := store.Images(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + if images, err := store.List("foo"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Path foo has wrong number of images (should be %d, not %d)", 1, l) + } else if images[0].Id != image.Id { + t.Fatalf("Imported image should be listed at path foo (%s != %s)", images[0], image) + } +} + +// Copy an image to a new path +func TestCopyNewPath(t *testing.T) { + store, err := TempStore("testcopynewpath") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + src, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + dst, err := src.Copy("bar") + if err != nil { + t.Fatal(err) + } + // ID should be the same + if src.Id != dst.Id { + t.Fatal("Different IDs") + } + // Check number of images at source path + if images, err := store.List("foo"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatal("Wrong number of images at source path (should be %d, not %d)", 1, l) + } + // Check number of images at destination path + if images, err := store.List("bar"); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatal("Wrong number of images at destination path (should be %d, not %d)", 1, l) + } + if err := healthCheck(store); err != nil { + t.Fatal(err) + } +} + +// Copying an image to the same path twice should fail +func TestCopySameName(t *testing.T) { + store, err := TempStore("testcopysamename") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + src, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + _, err = src.Copy("foo") + if err == nil { + t.Fatal("Copying an image to the same patch twice should fail.") + } +} + +/* +func TestMount(t *testing.T) { + store, err := TempStore() + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + // Create mount targets + root, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + rw, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + mountpoint, err := image.Mount(root, rw) + if err != nil { + t.Fatal(err) + } + defer mountpoint.Umount() + // Mountpoint should be marked as mounted + if !mountpoint.Mounted() { + t.Fatal("Mountpoint not mounted") + } + // There should be one mountpoint registered + if l := len(image.Mountpoints()); l != 1 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, l) + } + // Unmounting should work + if err := mountpoint.Umount(); err != nil { + t.Fatal(err) + } + // De-registering should work + if err := mountpoint.Deregister(); err != nil { + t.Fatal(err) + } + if l := len(image.Mountpoints()); l != 0 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, l) + } + // General health check + if err := healthCheck(); err != nil { + t.Fatal(err) + } +} +*/ + +func TempStore(prefix string) (*Store, error) { + dir, err := ioutil.TempDir("", "docker-fs-test-" + prefix) + if err != nil { + return nil, err + } + return New(dir) +} + +func nuke(store *Store) error { + return os.RemoveAll(store.Root) +} + +// Look for inconsistencies in a store. +func healthCheck(store *Store) error { + parents := make(map[string]bool) + paths, err := store.Paths() + if err != nil { + return err + } + for _, path := range paths { + images, err := store.List(path) + if err != nil { + return err + } + IDs := make(map[string]bool) // All IDs for this path + for _, img := range images { + // Check for duplicate IDs per path + if _, exists := IDs[img.Id]; exists { + return errors.New(fmt.Sprintf("Duplicate ID: %s", img.Id)) + } else { + IDs[img.Id] = true + } + // Store parent for 2nd pass + if parent := img.Parent; parent != "" { + parents[parent] = true + } + } + } + // Check non-existing parents + for parent := range parents { + if _, exists := parents[parent]; !exists { + return errors.New("Reference to non-registered parent: " + parent) + } + } + return nil +} + From 07b6bc3fc738ab3a27e9760b0efc57c78d7f98af Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 21:48:04 -0800 Subject: [PATCH 02/50] fs.store.Get(): use gorp.Get() instead of gorp.Select() --- fs/store.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/fs/store.go b/fs/store.go index e42959762..c34dbd0b4 100644 --- a/fs/store.go +++ b/fs/store.go @@ -83,14 +83,8 @@ func (store *Store) List(pth string) ([]*Image, error) { } func (store *Store) Get(id string) (*Image, error) { - images, err := store.orm.Select(Image{}, "select * from images where Id=?", id) - if err != nil { - return nil, err - } - if len(images) < 1 { - return nil, os.ErrNotExist - } - return images[0].(*Image), nil + img, err := store.orm.Get(Image{}, id) + return img.(*Image), err } func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (*Image, error) { From aab32e1012bd6061ec1049adc2bbe60ed1918f54 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 21:50:59 -0800 Subject: [PATCH 03/50] Fix store.Register() --- fs/store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/store.go b/fs/store.go index c34dbd0b4..02bfd47e8 100644 --- a/fs/store.go +++ b/fs/store.go @@ -115,6 +115,7 @@ func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (* } func (store *Store) Register(image *Image, pth string) error { + image.store = store // FIXME: import layer trans, err := store.orm.Begin() if err != nil { From 1531848ca6ead7c46497a625feba1224825e9011 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 15 Feb 2013 21:51:36 -0800 Subject: [PATCH 04/50] Getting started with mountpoints --- fs/store.go | 27 ++++++++++++++++++++++++++ fs/store_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/fs/store.go b/fs/store.go index 02bfd47e8..4ba23ae7d 100644 --- a/fs/store.go +++ b/fs/store.go @@ -29,6 +29,7 @@ func New(root string) (*Store, error) { orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") + orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") if err := orm.CreateTables(); err != nil { return nil, err } @@ -144,6 +145,32 @@ func (image *Image) Copy(pth string) (*Image, error) { return image, nil } +type Mountpoint struct { + Image string + Root string + Rw string +} + +func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { + mountpoint := &Mountpoint{Root: path.Clean(root), Rw: path.Clean(rw), Image: image.Id} + if err := image.store.orm.Insert(mountpoint); err != nil { + return nil, err + } + return mountpoint, nil +} + +func (image *Image) Mountpoints() ([]*Mountpoint, error) { + var mountpoints []*Mountpoint + res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) + if err != nil { + return nil, err + } + for _, mp := range res { + mountpoints = append(mountpoints, mp.(*Mountpoint)) + } + return mountpoints, nil +} + type Path struct { Path string diff --git a/fs/store_test.go b/fs/store_test.go index 940396227..89198f94d 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -113,6 +113,55 @@ func TestCopySameName(t *testing.T) { } } +func TestMountPoint(t *testing.T) { + store, err := TempStore("test-mountpoint") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + mountpoint, err := image.Mountpoint("/tmp/a", "/tmp/b") + if err != nil { + t.Fatal(err) + } + if mountpoint.Root != "/tmp/a" { + t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root) + } + if mountpoint.Rw!= "/tmp/b" { + t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw) + } +} + +func TestMountpointDuplicateRoot(t *testing.T) { + store, err := TempStore("test-mountpoint") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + _, err = image.Mountpoint("/tmp/a", "/tmp/b") + if err != nil { + t.Fatal(err) + } + if _, err = image.Mountpoint("/tmp/a", "/tmp/foobar"); err == nil { + t.Fatal("Duplicate mountpoint root should fail") + } +} + /* func TestMount(t *testing.T) { store, err := TempStore() From 6372a1a0d0b6506f25db7ba3dbba1e65fd6deb2d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 18 Feb 2013 15:25:43 -0800 Subject: [PATCH 05/50] docker/fs: initial support for filesystem layers (adapted from image/layers.go) --- {image => fs}/layers.go | 16 +++++---- fs/layers_test.go | 80 +++++++++++++++++++++++++++++++++++++++++ fs/store.go | 14 +++++++- 3 files changed, 103 insertions(+), 7 deletions(-) rename {image => fs}/layers.go (86%) create mode 100644 fs/layers_test.go diff --git a/image/layers.go b/fs/layers.go similarity index 86% rename from image/layers.go rename to fs/layers.go index f856ff81d..fa03ca586 100644 --- a/image/layers.go +++ b/fs/layers.go @@ -1,4 +1,4 @@ -package image +package fs import ( "errors" @@ -20,6 +20,10 @@ func NewLayerStore(root string) (*LayerStore, error) { if err != nil { return nil, err } + // Create the root directory if it doesn't exists + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { + return nil, err + } return &LayerStore{ Root: abspath, }, nil @@ -82,7 +86,10 @@ func (store *LayerStore) layerPath(id string) string { } -func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compression Compression) (string, error) { +func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, compression Compression) (string, error) { + if _, err := os.Stat(store.layerPath(id)); err == nil { + return "", errors.New("Layer already exists: " + id) + } tmp, err := store.Mktemp() defer os.RemoveAll(tmp) if err != nil { @@ -110,14 +117,11 @@ func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compressi } go io.Copy(stderr, untarStdout) untarCmd.Start() - hashR, hashW := io.Pipe() job_copy := future.Go(func() error { - _, err := io.Copy(io.MultiWriter(hashW, untarW), archive) - hashW.Close() + _, err := io.Copy(untarW, archive) untarW.Close() return err }) - id, err := future.ComputeId(hashR) if err != nil { return "", err } diff --git a/fs/layers_test.go b/fs/layers_test.go new file mode 100644 index 000000000..3d8e9e32a --- /dev/null +++ b/fs/layers_test.go @@ -0,0 +1,80 @@ +package fs + +import ( + "io/ioutil" + "testing" + "os" + "github.com/dotcloud/docker/fake" +) + + + +func TestLayersInit(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + // Root should exist + if _, err := os.Stat(store.Root); err != nil { + t.Fatal(err) + } + // List() should be empty + if l := store.List(); len(l) != 0 { + t.Fatalf("List() should return %d, not %d", 0, len(l)) + } +} + +func TestAddLayer(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + layer, err := store.AddLayer("foo", testArchive(t), os.Stderr, Uncompressed) + if err != nil { + t.Fatal(err) + } + // Layer path should exist + if _, err := os.Stat(layer); err != nil { + t.Fatal(err) + } + // List() should return 1 layer + if l := store.List(); len(l) != 1 { + t.Fatalf("List() should return %d elements, not %d", 1, len(l)) + } + // Get("foo") should return the correct layer + if foo := store.Get("foo"); foo != layer { + t.Fatalf("get(\"foo\") should return '%d', not '%d'", layer, foo) + } +} + +func TestAddLayerDuplicate(t *testing.T) { + store := tempStore(t) + defer os.RemoveAll(store.Root) + if _, err := store.AddLayer("foobar123", testArchive(t), os.Stderr, Uncompressed); err != nil { + t.Fatal(err) + } + if _, err := store.AddLayer("foobar123", testArchive(t), os.Stderr, Uncompressed); err == nil { + t.Fatalf("Creating duplicate layer should fail") + } +} + + +/* + * HELPER FUNCTIONS + */ + +func tempStore(t *testing.T) *LayerStore { + tmp, err := ioutil.TempDir("", "docker-fs-layerstore-") + if err != nil { + t.Fatal(err) + } + store, err := NewLayerStore(tmp) + if err != nil { + t.Fatal(err) + } + return store +} + +func testArchive(t *testing.T) Archive { + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + return archive +} diff --git a/fs/store.go b/fs/store.go index 4ba23ae7d..9bbc52e18 100644 --- a/fs/store.go +++ b/fs/store.go @@ -14,6 +14,7 @@ type Store struct { Root string db *sql.DB orm *gorp.DbMap + layers *LayerStore } type Archive io.Reader @@ -33,10 +34,15 @@ func New(root string) (*Store, error) { if err := orm.CreateTables(); err != nil { return nil, err } + layers, err := NewLayerStore(path.Join(root, "layers")) + if err != nil { + return nil, err + } return &Store{ Root: root, db: db, orm: orm, + layers: layers, }, nil } @@ -88,13 +94,19 @@ func (store *Store) Get(id string) (*Image, error) { return img.(*Image), err } -func (store *Store) Create(layer Archive, parent *Image, pth, comment string) (*Image, error) { +func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { // FIXME: actually do something with the layer... img := &Image{ Id : future.RandomId(), Comment: comment, store: store, } + // FIXME: we shouldn't have to pass os.Stderr to AddLayer()... + // FIXME: Archive should contain compression info. For now we only support uncompressed. + _, err := store.layers.AddLayer(img.Id, layerData, os.Stderr, Uncompressed) + if err != nil { + return nil, err + } path := &Path{ Path: path.Clean(pth), Image: img.Id, From 48974cc28d72598f0a35a4a6483b7006a1c18f9f Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 18 Feb 2013 21:09:25 -0800 Subject: [PATCH 06/50] FakeTar: /var/log/postgres can't be both a folder and a file --- fake/fake.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fake/fake.go b/fake/fake.go index a8c18a458..b3ec46310 100644 --- a/fake/fake.go +++ b/fake/fake.go @@ -14,7 +14,7 @@ func FakeTar() (io.Reader, error) { content := []byte("Hello world!\n") buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - for _, name := range []string {"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres", "/var/log/postgres/postgres.conf"} { + for _, name := range []string {"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { hdr := new(tar.Header) hdr.Size = int64(len(content)) hdr.Name = name From 8002af43fb54254ef786e4478ada7824e726baf7 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 18 Feb 2013 21:10:19 -0800 Subject: [PATCH 07/50] Better error context when failing to create a new image. Added basic tag support + unit tests --- fs/layers.go | 27 +++++++++++++++---------- fs/store.go | 52 +++++++++++++++++++++++++++++++++++++++++++----- fs/store_test.go | 31 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 15 deletions(-) diff --git a/fs/layers.go b/fs/layers.go index fa03ca586..f841b0809 100644 --- a/fs/layers.go +++ b/fs/layers.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "os/exec" + "fmt" "github.com/dotcloud/docker/future" ) @@ -15,6 +16,14 @@ type LayerStore struct { Root string } +type Compression uint32 + +const ( + Uncompressed Compression = iota + Bzip2 + Gzip +) + func NewLayerStore(root string) (*LayerStore, error) { abspath, err := filepath.Abs(root) if err != nil { @@ -93,7 +102,7 @@ func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, tmp, err := store.Mktemp() defer os.RemoveAll(tmp) if err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Mktemp failed: %s", err)) } extractFlags := "-x" if compression == Bzip2 { @@ -104,16 +113,16 @@ func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, untarCmd := exec.Command("tar", "-C", tmp, extractFlags) untarW, err := untarCmd.StdinPipe() if err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Could not obtain stdin pipe: %s", err)) } untarStderr, err := untarCmd.StderrPipe() if err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Could not obtain stderr pipe: %s", err)) } go io.Copy(stderr, untarStderr) untarStdout, err := untarCmd.StdoutPipe() if err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Could not obtain stdout pipe: %s", err)) } go io.Copy(stderr, untarStdout) untarCmd.Start() @@ -122,19 +131,17 @@ func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, untarW.Close() return err }) - if err != nil { - return "", err - } + if err := untarCmd.Wait(); err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Error while waiting for untar command to complete: %s", err)) } if err := <-job_copy; err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Error while copying: %s", err)) } layer := store.layerPath(id) if !store.Exists(id) { if err := os.Rename(tmp, layer); err != nil { - return "", err + return "", errors.New(fmt.Sprintf("Could not rename temp dir to layer %s: %s", layer, err)) } } return layer, nil diff --git a/fs/store.go b/fs/store.go index 9bbc52e18..d66fa4632 100644 --- a/fs/store.go +++ b/fs/store.go @@ -8,6 +8,8 @@ import ( "io" "path" "github.com/dotcloud/docker/future" + "fmt" + "errors" ) type Store struct { @@ -31,6 +33,7 @@ func New(root string) (*Store, error) { orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") + orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") if err := orm.CreateTables(); err != nil { return nil, err } @@ -105,7 +108,7 @@ func (store *Store) Create(layerData Archive, parent *Image, pth, comment string // FIXME: Archive should contain compression info. For now we only support uncompressed. _, err := store.layers.AddLayer(img.Id, layerData, os.Stderr, Uncompressed) if err != nil { - return nil, err + return nil, errors.New(fmt.Sprintf("Could not add layer: %s", err)) } path := &Path{ Path: path.Clean(pth), @@ -113,16 +116,16 @@ func (store *Store) Create(layerData Archive, parent *Image, pth, comment string } trans, err := store.orm.Begin() if err != nil { - return nil, err + return nil, errors.New(fmt.Sprintf("Could not begin transaction:", err)) } if err := trans.Insert(img); err != nil { - return nil, err + return nil, errors.New(fmt.Sprintf("Could not insert image info: %s", err)) } if err := trans.Insert(path); err != nil { - return nil, err + return nil, errors.New(fmt.Sprintf("Could not insert path info: %s", err)) } if err := trans.Commit(); err != nil { - return nil, err + return nil, errors.New(fmt.Sprintf("Could not commit transaction: %s", err)) } return img, nil } @@ -183,8 +186,47 @@ func (image *Image) Mountpoints() ([]*Mountpoint, error) { return mountpoints, nil } +func (store *Store) AddTag(imageId, tagName string) error { + if image, err := store.Get(imageId); err != nil { + return err + } else if image == nil { + return errors.New("No image with ID " + imageId) + } + + err2 := store.orm.Insert(&Tag{ + TagName: tagName, + Image: imageId, + }) + + return err2 +} + +func (store *Store) GetByTag(tagName string) (*Image, error) { + res, err := store.orm.Get(Tag{}, tagName) + if err != nil { + return nil, err + } else if res == nil { + return nil, errors.New("No image associated to tag \"" + tagName + "\"") + } + + tag := res.(*Tag) + + img, err2 := store.Get(tag.Image) + if err2 != nil { + return nil, err2 + } else if img == nil { + return nil, errors.New("Tag was found but image seems to be inexistent.") + } + + return img, nil +} type Path struct { Path string Image string } + +type Tag struct { + TagName string + Image string +} \ No newline at end of file diff --git a/fs/store_test.go b/fs/store_test.go index 89198f94d..289804940 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -52,6 +52,37 @@ func TestCreate(t *testing.T) { } } +func TestTag(t *testing.T) { + store, err := TempStore("testtag") + if err != nil { + t.Fatal(err) + } + defer nuke(store) + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + image, err := store.Create(archive, nil, "foo", "Testing") + if err != nil { + t.Fatal(err) + } + if images, err := store.Images(); err != nil { + t.Fatal(err) + } else if l := len(images); l != 1 { + t.Fatalf("Wrong number of images. Should be %d, not %d", 1, l) + } + + if err := store.AddTag(image.Id, "baz"); err != nil { + t.Fatalf("Error while adding a tag to created image: %s", err) + } + + if taggedImage, err := store.GetByTag("baz"); err != nil { + t.Fatalf("Error while trying to retrieve image for tag 'baz': %s", err) + } else if taggedImage.Id != image.Id { + t.Fatalf("Expected to retrieve image %s but found %s instead", image.Id, taggedImage.Id) + } +} + // Copy an image to a new path func TestCopyNewPath(t *testing.T) { store, err := TempStore("testcopynewpath") From 04bcdf9030123a0d59a930e072429664a0e11194 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 21 Feb 2013 23:01:13 -0800 Subject: [PATCH 08/50] Implementing mounting/unmounting of images. Currently missing Mounpoint#Deregister --- mount_darwin.go => fs/mount_darwin.go | 0 mount_linux.go => fs/mount_linux.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename mount_darwin.go => fs/mount_darwin.go (100%) rename mount_linux.go => fs/mount_linux.go (100%) diff --git a/mount_darwin.go b/fs/mount_darwin.go similarity index 100% rename from mount_darwin.go rename to fs/mount_darwin.go diff --git a/mount_linux.go b/fs/mount_linux.go similarity index 100% rename from mount_linux.go rename to fs/mount_linux.go From b4ea31e1deb44dc28c75aaaf24ff99c87c7f595a Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 21 Feb 2013 23:01:30 -0800 Subject: [PATCH 09/50] Implementing mounting/unmounting of images. Currently missing Mounpoint#Deregister --- fs/mount_darwin.go | 2 +- fs/mount_linux.go | 2 +- fs/store.go | 220 +++++++++++++++++++++++++++++++++++++-------- fs/store_test.go | 31 +++---- 4 files changed, 199 insertions(+), 56 deletions(-) diff --git a/fs/mount_darwin.go b/fs/mount_darwin.go index aeac78cda..540d6f769 100644 --- a/fs/mount_darwin.go +++ b/fs/mount_darwin.go @@ -1,4 +1,4 @@ -package docker +package fs import "errors" diff --git a/fs/mount_linux.go b/fs/mount_linux.go index a5a24e848..6dd2a39af 100644 --- a/fs/mount_linux.go +++ b/fs/mount_linux.go @@ -1,4 +1,4 @@ -package docker +package fs import "syscall" diff --git a/fs/store.go b/fs/store.go index d66fa4632..915e794a2 100644 --- a/fs/store.go +++ b/fs/store.go @@ -2,21 +2,24 @@ package fs import ( "database/sql" - _ "github.com/mattn/go-sqlite3" - "github.com/coopernurse/gorp" - "os" - "io" - "path" - "github.com/dotcloud/docker/future" - "fmt" "errors" + "fmt" + "github.com/coopernurse/gorp" + "github.com/dotcloud/docker/future" + _ "github.com/mattn/go-sqlite3" + "io" + "os" + "path" + "syscall" + "time" + "path/filepath" ) type Store struct { - Root string - db *sql.DB - orm *gorp.DbMap - layers *LayerStore + Root string + db *sql.DB + orm *gorp.DbMap + layers *LayerStore } type Archive io.Reader @@ -42,14 +45,14 @@ func New(root string) (*Store, error) { return nil, err } return &Store{ - Root: root, - db: db, - orm: orm, + Root: root, + db: db, + orm: orm, layers: layers, }, nil } -func (store *Store) imageList(src []interface{}) ([]*Image) { +func (store *Store) imageList(src []interface{}) []*Image { var images []*Image for _, i := range src { img := i.(*Image) @@ -60,7 +63,7 @@ func (store *Store) imageList(src []interface{}) ([]*Image) { } func (store *Store) Images() ([]*Image, error) { - images , err := store.orm.Select(Image{}, "select * from images") + images, err := store.orm.Select(Image{}, "select * from images") if err != nil { return nil, err } @@ -94,15 +97,18 @@ func (store *Store) List(pth string) ([]*Image, error) { func (store *Store) Get(id string) (*Image, error) { img, err := store.orm.Get(Image{}, id) + if img == nil { + return nil, err + } return img.(*Image), err } func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { // FIXME: actually do something with the layer... img := &Image{ - Id : future.RandomId(), - Comment: comment, - store: store, + Id: future.RandomId(), + Comment: comment, + store: store, } // FIXME: we shouldn't have to pass os.Stderr to AddLayer()... // FIXME: Archive should contain compression info. For now we only support uncompressed. @@ -111,8 +117,8 @@ func (store *Store) Create(layerData Archive, parent *Image, pth, comment string return nil, errors.New(fmt.Sprintf("Could not add layer: %s", err)) } path := &Path{ - Path: path.Clean(pth), - Image: img.Id, + Path: path.Clean(pth), + Image: img.Id, } trans, err := store.orm.Begin() if err != nil { @@ -142,14 +148,11 @@ func (store *Store) Register(image *Image, pth string) error { return trans.Commit() } - - - type Image struct { - Id string - Parent string - Comment string - store *Store `db:"-"` + Id string + Parent string + Comment string + store *Store `db:"-"` } @@ -161,19 +164,41 @@ func (image *Image) Copy(pth string) (*Image, error) { } type Mountpoint struct { - Image string - Root string - Rw string + Image string + Root string + Rw string } func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { - mountpoint := &Mountpoint{Root: path.Clean(root), Rw: path.Clean(rw), Image: image.Id} + mountpoint := &Mountpoint{ + Root: path.Clean(root), + Rw: path.Clean(rw), + Image: image.Id, + } if err := image.store.orm.Insert(mountpoint); err != nil { return nil, err } return mountpoint, nil } +func (image *Image) layers() ([]string, error) { + var list []string + var err error + currentImg := image + for currentImg != nil { + if layer := image.store.layers.Get(image.Id); layer != "" { + list = append(list, layer) + } else { + return list, fmt.Errorf("Layer not found for image %s", image.Id) + } + currentImg, err = currentImg.store.Get(currentImg.Parent) + if err != nil { + return list, fmt.Errorf("Error while getting parent image: %v", err) + } + } + return list, nil +} + func (image *Image) Mountpoints() ([]*Mountpoint, error) { var mountpoints []*Mountpoint res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) @@ -186,6 +211,123 @@ func (image *Image) Mountpoints() ([]*Mountpoint, error) { return mountpoints, nil } +func (image *Image) Mount(root, rw string) (*Mountpoint, error) { + var mountpoint *Mountpoint + if mp, err := image.fetchMountpoint(root, rw); err != nil { + return nil, err + } else if mp == nil { + mountpoint, err = image.Mountpoint(root, rw) + if err != nil { + return nil, fmt.Errorf("Could not create mountpoint: %s", err) + } else if mountpoint == nil { + return nil, errors.New("No mountpoint created") + } + } else { + mountpoint = mp + } + + if err := mountpoint.createFolders(); err != nil { + return nil, err + } + + // FIXME: Now mount the layers + rwBranch := fmt.Sprintf("%v=rw", mountpoint.Rw) + roBranches := "" + layers, err := image.layers() + if err != nil { + return nil, err + } + + for _, layer := range layers { + roBranches += fmt.Sprintf("%v=ro:", layer) + } + branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) + if err := mount("none", mountpoint.Root, "aufs", 0, branches); err != nil { + return mountpoint, err + } + if !mountpoint.Mounted() { + return mountpoint, errors.New("Mount failed") + } + + return mountpoint, nil +} + +func (mp *Mountpoint) createFolders() error { + if err := os.Mkdir(mp.Root, 0755); err != nil && !os.IsExist(err) { + return err + } + if err := os.Mkdir(mp.Rw, 0755); err != nil && !os.IsExist(err) { + return err + } + return nil +} + +func (mp *Mountpoint) Mounted() bool { + root, err := os.Stat(mp.Root) + if err != nil { + if os.IsNotExist(err) { + return false + } + panic(err) + } + parent, err := os.Stat(filepath.Join(mp.Root, "..")) + if err != nil { + panic(err) + } + + rootSt := root.Sys().(*syscall.Stat_t) + parentSt := parent.Sys().(*syscall.Stat_t) + return rootSt.Dev != parentSt.Dev +} + +func (mp *Mountpoint) Umount() error { + if !mp.Mounted() { + return errors.New("Mountpoint doesn't seem to be mounted") + } + if err := syscall.Unmount(mp.Root, 0); err != nil { + return fmt.Errorf("Unmount syscall failed: %v", err) + } + if mp.Mounted() { + return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", mp.Root) + } + // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint + // for some time. We'll just keep retrying until it succeeds. + for retries := 0; retries < 1000; retries++ { + err := os.Remove(mp.Root) + if err == nil { + // rm mntpoint succeeded + return nil + } + if os.IsNotExist(err) { + // mntpoint doesn't exist anymore. Success. + return nil + } + // fmt.Printf("(%v) Remove %v returned: %v\n", retries, mp.Root, err) + time.Sleep(10 * time.Millisecond) + } + return fmt.Errorf("Umount: Failed to umount %v", mp.Root) + +} + +func (mp *Mountpoint) Deregister() error { + if mp.Mounted() { + return errors.New("Mountpoint is currently mounted, can't deregister") + } + + return errors.New("Not yet implemented") +} + +func (image *Image) fetchMountpoint(root, rw string) (*Mountpoint, error) { + res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=? and Root=? and Rw=?", image.Id, root, rw) + if err != nil { + return nil, err + } else if len(res) < 1 || res[0] == nil { + return nil, nil + } + + return res[0].(*Mountpoint), nil +} + func (store *Store) AddTag(imageId, tagName string) error { if image, err := store.Get(imageId); err != nil { return err @@ -194,8 +336,8 @@ func (store *Store) AddTag(imageId, tagName string) error { } err2 := store.orm.Insert(&Tag{ - TagName: tagName, - Image: imageId, + TagName: tagName, + Image: imageId, }) return err2 @@ -222,11 +364,11 @@ func (store *Store) GetByTag(tagName string) (*Image, error) { } type Path struct { - Path string - Image string + Path string + Image string } type Tag struct { - TagName string - Image string -} \ No newline at end of file + TagName string + Image string +} diff --git a/fs/store_test.go b/fs/store_test.go index 289804940..f2cc06534 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -1,12 +1,12 @@ package fs import ( - "testing" - "io/ioutil" - "github.com/dotcloud/docker/fake" - "os" "errors" "fmt" + "github.com/dotcloud/docker/fake" + "io/ioutil" + "os" + "testing" ) func TestInit(t *testing.T) { @@ -165,7 +165,7 @@ func TestMountPoint(t *testing.T) { if mountpoint.Root != "/tmp/a" { t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/a", mountpoint.Root) } - if mountpoint.Rw!= "/tmp/b" { + if mountpoint.Rw != "/tmp/b" { t.Fatal("Wrong mountpoint root (should be %s, not %s)", "/tmp/b", mountpoint.Rw) } } @@ -193,9 +193,8 @@ func TestMountpointDuplicateRoot(t *testing.T) { } } -/* func TestMount(t *testing.T) { - store, err := TempStore() + store, err := TempStore("test-mount") if err != nil { t.Fatal(err) } @@ -227,8 +226,10 @@ func TestMount(t *testing.T) { t.Fatal("Mountpoint not mounted") } // There should be one mountpoint registered - if l := len(image.Mountpoints()); l != 1 { - t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, l) + if mps, err := image.Mountpoints(); err != nil { + t.Fatal(err) + } else if len(mps) != 1 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 1, len(mps)) } // Unmounting should work if err := mountpoint.Umount(); err != nil { @@ -238,18 +239,19 @@ func TestMount(t *testing.T) { if err := mountpoint.Deregister(); err != nil { t.Fatal(err) } - if l := len(image.Mountpoints()); l != 0 { - t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, l) + if mps, err := image.Mountpoints(); err != nil { + t.Fatal(err) + } else if len(mps) != 0 { + t.Fatal("Wrong number of mountpoints registered (should be %d, not %d)", 0, len(mps)) } // General health check - if err := healthCheck(); err != nil { + if err := healthCheck(store); err != nil { t.Fatal(err) } } -*/ func TempStore(prefix string) (*Store, error) { - dir, err := ioutil.TempDir("", "docker-fs-test-" + prefix) + dir, err := ioutil.TempDir("", "docker-fs-test-"+prefix) if err != nil { return nil, err } @@ -294,4 +296,3 @@ func healthCheck(store *Store) error { } return nil } - From 8b61af189597579f1086a42fcdc824bfe737dce7 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 21 Feb 2013 23:07:10 -0800 Subject: [PATCH 10/50] Re-added mount_*.go in docker package to not break tests/build --- mount_darwin.go | 7 +++++++ mount_linux.go | 8 ++++++++ 2 files changed, 15 insertions(+) create mode 100644 mount_darwin.go create mode 100644 mount_linux.go diff --git a/mount_darwin.go b/mount_darwin.go new file mode 100644 index 000000000..aeac78cda --- /dev/null +++ b/mount_darwin.go @@ -0,0 +1,7 @@ +package docker + +import "errors" + +func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { + return errors.New("mount is not implemented on darwin") +} diff --git a/mount_linux.go b/mount_linux.go new file mode 100644 index 000000000..a5a24e848 --- /dev/null +++ b/mount_linux.go @@ -0,0 +1,8 @@ +package docker + +import "syscall" + + +func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { + return syscall.Mount(source, target, fstype, flags, data) +} From 2ebf3464a1ed5bf32e33eff45cfa8e8394b5cd43 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 26 Feb 2013 17:45:46 -0800 Subject: [PATCH 11/50] Halfway through fs branch fixes, TestUser not passing --- container.go | 25 ++-- container_test.go | 46 ++++---- docker.go | 27 +++-- docker_test.go | 57 ++++++++-- dockerd/dockerweb.html | 0 filesystem.go | 252 ----------------------------------------- filesystem_test.go | 208 ---------------------------------- fs/layers.go | 2 +- fs/store.go | 37 +++++- fs/store_test.go | 2 +- lxc_template.go | 2 +- 11 files changed, 143 insertions(+), 515 deletions(-) mode change 100644 => 100755 dockerd/dockerweb.html delete mode 100644 filesystem.go delete mode 100644 filesystem_test.go diff --git a/container.go b/container.go index 57ec53173..ca7ab37f2 100644 --- a/container.go +++ b/container.go @@ -14,6 +14,7 @@ import ( "strings" "syscall" "time" + "./fs" ) var sysInitPath string @@ -32,7 +33,7 @@ type Container struct { Args []string Config *Config - Filesystem *Filesystem + Mountpoint *fs.Mountpoint State *State SysInitPath string @@ -55,7 +56,11 @@ type Config struct { OpenStdin bool // Open stdin } -func createContainer(id string, root string, command string, args []string, layers []string, config *Config) (*Container, error) { +func createContainer(id string, root string, command string, args []string, image *fs.Image, config *Config) (*Container, error) { + mountpoint, err := image.Mountpoint(path.Join(root, "rootfs"), path.Join(root, "rw")) + if err != nil { + return nil, err + } container := &Container{ Id: id, Root: root, @@ -63,7 +68,7 @@ func createContainer(id string, root string, command string, args []string, laye Path: command, Args: args, Config: config, - Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), + Mountpoint: mountpoint, State: newState(), SysInitPath: sysInitPath, @@ -84,9 +89,9 @@ func createContainer(id string, root string, command string, args []string, laye if err := os.Mkdir(root, 0700); err != nil { return nil, err } - if err := container.Filesystem.createMountPoints(); err != nil { + /*if err := container.Filesystem.createMountPoints(); err != nil { return nil, err - } + }*/ if err := container.save(); err != nil { return nil, err } @@ -108,9 +113,9 @@ func loadContainer(containerPath string) (*Container, error) { if err := json.Unmarshal(data, container); err != nil { return nil, err } - if err := container.Filesystem.createMountPoints(); err != nil { - return nil, err - } + // if err := container.Filesystem.createMountPoints(); err != nil { + // return nil, err + // } if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() } else { @@ -260,7 +265,7 @@ func (container *Container) start() error { } func (container *Container) Start() error { - if err := container.Filesystem.EnsureMounted(); err != nil { + if err := container.Mountpoint.EnsureMounted(); err != nil { return err } if err := container.generateLXCConfig(); err != nil { @@ -351,7 +356,7 @@ func (container *Container) monitor() { // Cleanup container.stdout.Close() container.stderr.Close() - if err := container.Filesystem.Umount(); err != nil { + if err := container.Mountpoint.Umount(); err != nil { log.Printf("%v: Failed to umount filesystem: %v", container.Id, err) } diff --git a/container_test.go b/container_test.go index 17ec85c6b..bcf450c73 100644 --- a/container_test.go +++ b/container_test.go @@ -18,7 +18,7 @@ func TestStart(t *testing.T) { "start_test", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ Ram: 33554432, }, @@ -54,7 +54,7 @@ func TestRun(t *testing.T) { "run_test", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ Ram: 33554432, }, @@ -84,7 +84,7 @@ func TestOutput(t *testing.T) { "output_test", "echo", []string{"-n", "foobar"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -109,7 +109,7 @@ func TestKill(t *testing.T) { "stop_test", "cat", []string{"/dev/zero"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -152,7 +152,7 @@ func TestExitCode(t *testing.T) { "exit_test_1", "/bin/true", []string{""}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -167,7 +167,7 @@ func TestExitCode(t *testing.T) { "exit_test_2", "/bin/false", []string{""}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -196,7 +196,7 @@ func TestRestart(t *testing.T) { "restart_test", "echo", []string{"-n", "foobar"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -230,7 +230,7 @@ func TestRestartStdin(t *testing.T) { "restart_stdin_test", "cat", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ OpenStdin: true, }, @@ -281,7 +281,7 @@ func TestUser(t *testing.T) { "user_default", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -301,7 +301,7 @@ func TestUser(t *testing.T) { "user_root", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "root", }, @@ -323,7 +323,7 @@ func TestUser(t *testing.T) { "user_uid0", "id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "0", }, @@ -343,9 +343,9 @@ func TestUser(t *testing.T) { // Set a different user by uid container, err = docker.Create( "user_uid1", - "id", + "/usr/bin/id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "1", }, @@ -355,8 +355,10 @@ func TestUser(t *testing.T) { } defer docker.Destroy(container) output, err = container.Output() - if err != nil || container.State.ExitCode != 0 { + if err != nil { t.Fatal(err) + } else if container.State.ExitCode != 0 { + t.Fatalf("Container exit code is invalid: %d\nOutput:\n%s\n", container.State.ExitCode, output) } if !strings.Contains(string(output), "uid=1(daemon) gid=1(daemon)") { t.Error(string(output)) @@ -365,9 +367,9 @@ func TestUser(t *testing.T) { // Set a different user by username container, err = docker.Create( "user_daemon", - "id", + "/usr/bin/id", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ User: "daemon", }, @@ -395,7 +397,7 @@ func TestMultipleContainers(t *testing.T) { "container1", "cat", []string{"/dev/zero"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -407,7 +409,7 @@ func TestMultipleContainers(t *testing.T) { "container2", "cat", []string{"/dev/zero"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -450,7 +452,7 @@ func TestStdin(t *testing.T) { "stdin_test", "cat", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ OpenStdin: true, }, @@ -485,7 +487,7 @@ func TestTty(t *testing.T) { "tty_test", "cat", []string{}, - []string{testLayerPath}, + GetTestImage(docker), &Config{ OpenStdin: true, }, @@ -521,7 +523,7 @@ func BenchmarkRunSequencial(b *testing.B) { fmt.Sprintf("bench_%v", i), "echo", []string{"-n", "foo"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -557,7 +559,7 @@ func BenchmarkRunParallel(b *testing.B) { fmt.Sprintf("bench_%v", i), "echo", []string{"-n", "foo"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { diff --git a/docker.go b/docker.go index 2b334976d..0885de380 100644 --- a/docker.go +++ b/docker.go @@ -8,12 +8,14 @@ import ( "os" "path" "sort" + "./fs" ) type Docker struct { - root string - repository string - containers *list.List + root string + repository string + containers *list.List + Store *fs.Store } func (docker *Docker) List() []*Container { @@ -46,12 +48,12 @@ func (docker *Docker) Exists(id string) bool { return docker.Get(id) != nil } -func (docker *Docker) Create(id string, command string, args []string, layers []string, config *Config) (*Container, error) { +func (docker *Docker) Create(id string, command string, args []string, image *fs.Image, config *Config) (*Container, error) { if docker.Exists(id) { return nil, fmt.Errorf("Container %v already exists", id) } root := path.Join(docker.repository, id) - container, err := createContainer(id, root, command, args, layers, config) + container, err := createContainer(id, root, command, args, image, config) if err != nil { return nil, err } @@ -68,10 +70,14 @@ func (docker *Docker) Destroy(container *Container) error { if err := container.Stop(); err != nil { return err } - if container.Filesystem.IsMounted() { - if err := container.Filesystem.Umount(); err != nil { + if container.Mountpoint.Mounted() { + if err := container.Mountpoint.Umount(); err != nil { log.Printf("Unable to umount container %v: %v", container.Id, err) } + + if err := container.Mountpoint.Deregister(); err != nil { + log.Printf("Unable to deregiser mountpoint %v: %v", container.Mountpoint.Root, err) + } } if err := os.RemoveAll(container.Root); err != nil { log.Printf("Unable to remove filesystem for %v: %v", container.Id, err) @@ -91,6 +97,7 @@ func (docker *Docker) restore() error { log.Printf("Failed to load container %v: %v", v.Name(), err) continue } + container.Mountpoint.Store = docker.Store docker.containers.PushBack(container) } return nil @@ -101,10 +108,16 @@ func New() (*Docker, error) { } func NewFromDirectory(root string) (*Docker, error) { + store, err := fs.New(path.Join(root, "images")) + if err != nil { + return nil, err + } + docker := &Docker{ root: root, repository: path.Join(root, "containers"), containers: list.New(), + Store: store, } if err := os.Mkdir(docker.repository, 0700); err != nil && !os.IsExist(err) { diff --git a/docker_test.go b/docker_test.go index d786da950..c612c4f60 100644 --- a/docker_test.go +++ b/docker_test.go @@ -5,9 +5,20 @@ import ( "log" "os" "testing" + "io" + "./fs" ) -const testLayerPath string = "/var/lib/docker/images/docker-ut" +const testLayerPath string = "/var/lib/docker/docker-ut.tar" + +func layerArchive(tarfile string) (io.Reader, error) { + // FIXME: need to close f somewhere + f, err := os.Open(tarfile) + if err != nil { + return nil, err + } + return f, nil +} func init() { // Hack to run sys init during unit testing @@ -21,6 +32,7 @@ func init() { panic(err) } log.Fatalf("Unit test base image not found. Please fix the problem by running \"debootstrap --arch=amd64 quantal %v\"", testLayerPath) + return } } @@ -33,9 +45,28 @@ func newTestDocker() (*Docker, error) { if err != nil { return nil, err } + + if layer, err := layerArchive(testLayerPath); err != nil { + panic(err) + } else { + _, err = docker.Store.Create(layer, nil, "docker-ut", "unit tests") + if err != nil { + panic(err) + } + } return docker, nil } +func GetTestImage(docker *Docker) (*fs.Image) { + imgs, err := docker.Store.Images() + if err != nil { + panic(err) + } else if len(imgs) < 1 { + panic("GASP") + } + return imgs[0] +} + func TestCreate(t *testing.T) { docker, err := newTestDocker() if err != nil { @@ -50,7 +81,7 @@ func TestCreate(t *testing.T) { "test_create", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -98,7 +129,7 @@ func TestDestroy(t *testing.T) { "test_destroy", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -146,7 +177,7 @@ func TestGet(t *testing.T) { "test1", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -158,7 +189,7 @@ func TestGet(t *testing.T) { "test2", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -170,7 +201,7 @@ func TestGet(t *testing.T) { "test3", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker), &Config{}, ) if err != nil { @@ -198,16 +229,26 @@ func TestRestore(t *testing.T) { t.Fatal(err) } - // Create a container with one instance of docker docker1, err := NewFromDirectory(root) if err != nil { t.Fatal(err) } + + if layer, err := layerArchive(testLayerPath); err != nil { + panic(err) + } else { + _, err = docker1.Store.Create(layer, nil, "docker-ut", "unit tests") + if err != nil { + panic(err) + } + } + + // Create a container with one instance of docker container1, err := docker1.Create( "restore_test", "ls", []string{"-al"}, - []string{testLayerPath}, + GetTestImage(docker1), &Config{}, ) if err != nil { diff --git a/dockerd/dockerweb.html b/dockerd/dockerweb.html old mode 100644 new mode 100755 diff --git a/filesystem.go b/filesystem.go deleted file mode 100644 index c775fa0c1..000000000 --- a/filesystem.go +++ /dev/null @@ -1,252 +0,0 @@ -package docker - -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "syscall" - "time" -) - -type Filesystem struct { - RootFS string - RWPath string - // The layers to be mounted on top of each other via aufs. - // Layers are ordered top-to-bottom: the first layer in the list will be mounted on top of the others. - // In other words, THE BASE IMAGE SHOULD BE LAST! - Layers []string -} - -func (fs *Filesystem) createMountPoints() error { - if err := os.Mkdir(fs.RootFS, 0755); err != nil && !os.IsExist(err) { - return err - } - if err := os.Mkdir(fs.RWPath, 0755); err != nil && !os.IsExist(err) { - return err - } - return nil -} - -func (fs *Filesystem) Mount() error { - if fs.IsMounted() { - return errors.New("Mount: Filesystem already mounted") - } - if err := fs.createMountPoints(); err != nil { - return err - } - rwBranch := fmt.Sprintf("%v=rw", fs.RWPath) - roBranches := "" - for _, layer := range fs.Layers { - roBranches += fmt.Sprintf("%v=ro:", layer) - } - branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) - if err := mount("none", fs.RootFS, "aufs", 0, branches); err != nil { - return err - } - if !fs.IsMounted() { - return errors.New("Mount failed") - } - return nil -} - -func (fs *Filesystem) Umount() error { - if !fs.IsMounted() { - return errors.New("Umount: Filesystem not mounted") - } - if err := syscall.Unmount(fs.RootFS, 0); err != nil { - return err - } - if fs.IsMounted() { - return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", fs.RootFS) - } - // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint - // for some time. We'll just keep retrying until it succeeds. - for retries := 0; retries < 1000; retries++ { - err := os.Remove(fs.RootFS) - if err == nil { - // rm mntpoint succeeded - return nil - } - if os.IsNotExist(err) { - // mntpoint doesn't exist anymore. Success. - return nil - } - // fmt.Printf("(%v) Remove %v returned: %v\n", retries, fs.RootFS, err) - time.Sleep(10 * time.Millisecond) - } - return fmt.Errorf("Umount: Failed to umount %v", fs.RootFS) -} - -func (fs *Filesystem) IsMounted() bool { - mntpoint, err := os.Stat(fs.RootFS) - if err != nil { - if os.IsNotExist(err) { - return false - } - panic(err) - } - parent, err := os.Stat(filepath.Join(fs.RootFS, "..")) - if err != nil { - panic(err) - } - - mntpointSt := mntpoint.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) - return mntpointSt.Dev != parentSt.Dev -} - -// Tar returns the contents of the filesystem as an uncompressed tar stream -func (fs *Filesystem) Tar() (io.Reader, error) { - if err := fs.EnsureMounted(); err != nil { - return nil, err - } - return Tar(fs.RootFS) -} - -func (fs *Filesystem) EnsureMounted() error { - if !fs.IsMounted() { - if err := fs.Mount(); err != nil { - return err - } - } - return nil -} - -type ChangeType int - -const ( - ChangeModify = iota - ChangeAdd - ChangeDelete -) - -type Change struct { - Path string - Kind ChangeType -} - -func (change *Change) String() string { - var kind string - switch change.Kind { - case ChangeModify: - kind = "C" - case ChangeAdd: - kind = "A" - case ChangeDelete: - kind = "D" - } - return fmt.Sprintf("%s %s", kind, change.Path) -} - -func (fs *Filesystem) Changes() ([]Change, error) { - var changes []Change - err := filepath.Walk(fs.RWPath, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - - // Rebase path - path, err = filepath.Rel(fs.RWPath, path) - if err != nil { - return err - } - path = filepath.Join("/", path) - - // Skip root - if path == "/" { - return nil - } - - // Skip AUFS metadata - if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { - return err - } - - change := Change{ - Path: path, - } - - // Find out what kind of modification happened - file := filepath.Base(path) - // If there is a whiteout, then the file was removed - if strings.HasPrefix(file, ".wh.") { - originalFile := strings.TrimLeft(file, ".wh.") - change.Path = filepath.Join(filepath.Dir(path), originalFile) - change.Kind = ChangeDelete - } else { - // Otherwise, the file was added - change.Kind = ChangeAdd - - // ...Unless it already existed in a top layer, in which case, it's a modification - for _, layer := range fs.Layers { - stat, err := os.Stat(filepath.Join(layer, path)) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - // The file existed in the top layer, so that's a modification - - // However, if it's a directory, maybe it wasn't actually modified. - // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar - if stat.IsDir() && f.IsDir() { - if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { - // Both directories are the same, don't record the change - return nil - } - } - change.Kind = ChangeModify - break - } - } - } - - // Record change - changes = append(changes, change) - return nil - }) - if err != nil { - return nil, err - } - return changes, nil -} - -// Reset removes all changes to the filesystem, reverting it to its initial state. -func (fs *Filesystem) Reset() error { - if err := os.RemoveAll(fs.RWPath); err != nil { - return err - } - // We removed the RW directory itself along with its content: let's re-create an empty one. - if err := fs.createMountPoints(); err != nil { - return err - } - return nil -} - -// Open opens the named file for reading. -func (fs *Filesystem) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { - if err := fs.EnsureMounted(); err != nil { - return nil, err - } - return os.OpenFile(filepath.Join(fs.RootFS, path), flag, perm) -} - -// ReadDir reads the directory named by dirname, relative to the Filesystem's root, -// and returns a list of sorted directory entries -func (fs *Filesystem) ReadDir(dirname string) ([]os.FileInfo, error) { - if err := fs.EnsureMounted(); err != nil { - return nil, err - } - return ioutil.ReadDir(filepath.Join(fs.RootFS, dirname)) -} - -func newFilesystem(rootfs string, rwpath string, layers []string) *Filesystem { - return &Filesystem{ - RootFS: rootfs, - RWPath: rwpath, - Layers: layers, - } -} diff --git a/filesystem_test.go b/filesystem_test.go deleted file mode 100644 index 5f24ae62a..000000000 --- a/filesystem_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package docker - -import ( - "bytes" - "io/ioutil" - "os" - "path" - "testing" -) - -func newTestFilesystem(t *testing.T, layers []string) (rootfs string, fs *Filesystem) { - rootfs, err := ioutil.TempDir("", "docker-test-root") - if err != nil { - t.Fatal(err) - } - rwpath, err := ioutil.TempDir("", "docker-test-rw") - if err != nil { - t.Fatal(err) - } - fs = newFilesystem(rootfs, rwpath, layers) - return -} - -func TestFilesystem(t *testing.T) { - _, filesystem := newTestFilesystem(t, []string{testLayerPath}) - if err := filesystem.Umount(); err == nil { - t.Errorf("Umount succeeded even though the filesystem was not mounted") - } - - if filesystem.IsMounted() { - t.Fatal("Filesystem should not be mounted") - } - - if err := filesystem.Mount(); err != nil { - t.Fatal(err) - } - - if !filesystem.IsMounted() { - t.Fatal("Filesystem should be mounted") - } - - if err := filesystem.Mount(); err == nil { - t.Errorf("Double mount succeeded") - } - - if !filesystem.IsMounted() { - t.Fatal("Filesystem should be mounted") - } - - if err := filesystem.Umount(); err != nil { - t.Fatal(err) - } - - if filesystem.IsMounted() { - t.Fatal("Filesystem should not be mounted") - } - - if err := filesystem.Umount(); err == nil { - t.Errorf("Umount succeeded even though the filesystem was already umounted") - } - - if filesystem.IsMounted() { - t.Fatal("Filesystem should not be mounted") - } -} - -func TestFilesystemMultiLayer(t *testing.T) { - // Create a fake layer - fakeLayer, err := ioutil.TempDir("", "docker-layer") - if err != nil { - t.Fatal(err) - } - data := []byte("hello world") - if err := ioutil.WriteFile(path.Join(fakeLayer, "test_file"), data, 0700); err != nil { - t.Fatal(err) - } - - // Create the layered filesystem and add our fake layer on top - rootfs, filesystem := newTestFilesystem(t, []string{testLayerPath, fakeLayer}) - - // Mount it - if err := filesystem.Mount(); err != nil { - t.Fatal(err) - } - defer func() { - if err := filesystem.Umount(); err != nil { - t.Fatal(err) - } - }() - - // Check to see whether we can access our fake layer - if _, err := os.Stat(path.Join(rootfs, "test_file")); err != nil { - t.Fatal(err) - } - fsdata, err := ioutil.ReadFile(path.Join(rootfs, "test_file")) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, fsdata) { - t.Error(string(fsdata)) - } -} - -func TestChanges(t *testing.T) { - rootfs, filesystem := newTestFilesystem(t, []string{testLayerPath}) - // Mount it - if err := filesystem.Mount(); err != nil { - t.Fatal(err) - } - defer filesystem.Umount() - - var changes []Change - var err error - - // Test without changes - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 0 { - t.Errorf("Unexpected changes :%v", changes) - } - - // Test simple change - file, err := os.Create(path.Join(rootfs, "test_change")) - if err != nil { - t.Fatal(err) - } - file.Close() - - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 1 { - t.Errorf("Unexpected changes :%v", changes) - } - if changes[0].Path != "/test_change" || changes[0].Kind != ChangeAdd { - t.Errorf("Unexpected changes :%v", changes) - } - - // Test subdirectory change - if err := os.Mkdir(path.Join(rootfs, "sub_change"), 0700); err != nil { - t.Fatal(err) - } - - file, err = os.Create(path.Join(rootfs, "sub_change", "test")) - if err != nil { - t.Fatal(err) - } - file.Close() - - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 3 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[0].Path != "/sub_change" || changes[0].Kind != ChangeAdd || changes[1].Path != "/sub_change/test" || changes[1].Kind != ChangeAdd { - t.Errorf("Unexpected changes: %v", changes) - } - - // Test permission change - if err := os.Chmod(path.Join(rootfs, "root"), 0000); err != nil { - t.Fatal(err) - } - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 4 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[0].Path != "/root" || changes[0].Kind != ChangeModify { - t.Errorf("Unexpected changes: %v", changes) - } - - // Test removal - if err := os.Remove(path.Join(rootfs, "etc", "passwd")); err != nil { - t.Fatal(err) - } - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 6 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[0].Path != "/etc" || changes[0].Kind != ChangeModify || changes[1].Path != "/etc/passwd" || changes[1].Kind != ChangeDelete { - t.Errorf("Unexpected changes: %v", changes) - } - - // Test sub-directory removal - if err := os.Remove(path.Join(rootfs, "usr", "bin", "sudo")); err != nil { - t.Fatal(err) - } - changes, err = filesystem.Changes() - if err != nil { - t.Fatal(err) - } - if len(changes) != 8 { - t.Errorf("Unexpected changes: %v", changes) - } - if changes[6].Path != "/usr/bin" || changes[6].Kind != ChangeModify || changes[7].Path != "/usr/bin/sudo" || changes[7].Kind != ChangeDelete { - t.Errorf("Unexpected changes: %v", changes) - } -} diff --git a/fs/layers.go b/fs/layers.go index f841b0809..bbb0ee201 100644 --- a/fs/layers.go +++ b/fs/layers.go @@ -9,7 +9,7 @@ import ( "os" "os/exec" "fmt" - "github.com/dotcloud/docker/future" + "../future" ) type LayerStore struct { diff --git a/fs/store.go b/fs/store.go index 915e794a2..9dd30e450 100644 --- a/fs/store.go +++ b/fs/store.go @@ -25,8 +25,12 @@ type Store struct { type Archive io.Reader func New(root string) (*Store, error) { + isNewStore := true + if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { return nil, err + } else if os.IsExist(err) { + isNewStore = false } db, err := sql.Open("sqlite3", path.Join(root, "db")) if err != nil { @@ -37,9 +41,12 @@ func New(root string) (*Store, error) { orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") - if err := orm.CreateTables(); err != nil { - return nil, err + if isNewStore { + if err := orm.CreateTables(); err != nil { + return nil, err + } } + layers, err := NewLayerStore(path.Join(root, "layers")) if err != nil { return nil, err @@ -100,7 +107,9 @@ func (store *Store) Get(id string) (*Image, error) { if img == nil { return nil, err } - return img.(*Image), err + res := img.(*Image) + res.store = store + return res, err } func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { @@ -167,6 +176,7 @@ type Mountpoint struct { Image string Root string Rw string + Store *Store `db:"-"` } func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { @@ -174,6 +184,7 @@ func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { Root: path.Clean(root), Rw: path.Clean(rw), Image: image.Id, + Store: image.store, } if err := image.store.orm.Insert(mountpoint); err != nil { return nil, err @@ -252,6 +263,19 @@ func (image *Image) Mount(root, rw string) (*Mountpoint, error) { return mountpoint, nil } +func (mp *Mountpoint) EnsureMounted() error { + if mp.Mounted() { + return nil + } + img, err := mp.Store.Get(mp.Image) + if err != nil { + return err + } + + _, err = img.Mount(mp.Root, mp.Rw) + return err +} + func (mp *Mountpoint) createFolders() error { if err := os.Mkdir(mp.Root, 0755); err != nil && !os.IsExist(err) { return err @@ -314,7 +338,8 @@ func (mp *Mountpoint) Deregister() error { return errors.New("Mountpoint is currently mounted, can't deregister") } - return errors.New("Not yet implemented") + _, err := mp.Store.orm.Delete(mp) + return err } func (image *Image) fetchMountpoint(root, rw string) (*Mountpoint, error) { @@ -325,7 +350,9 @@ func (image *Image) fetchMountpoint(root, rw string) (*Mountpoint, error) { return nil, nil } - return res[0].(*Mountpoint), nil + mp := res[0].(*Mountpoint) + mp.Store = image.store + return mp, nil } func (store *Store) AddTag(imageId, tagName string) error { diff --git a/fs/store_test.go b/fs/store_test.go index f2cc06534..954c83d86 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -3,7 +3,7 @@ package fs import ( "errors" "fmt" - "github.com/dotcloud/docker/fake" + "../fake" "io/ioutil" "os" "testing" diff --git a/lxc_template.go b/lxc_template.go index 4ac72da27..8b86eea0f 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -22,7 +22,7 @@ lxc.utsname = {{.Id}} #lxc.network.ipv4 = {ip_address}/{ip_prefix_len} # root filesystem -{{$ROOTFS := .Filesystem.RootFS}} +{{$ROOTFS := .Mountpoint.Root}} lxc.rootfs = {{$ROOTFS}} # use a dedicated pts for the container (and limit the number of pseudo terminal From 93ba6dd82bb62dbb9cac729cfa6626c469a87d00 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 8 Mar 2013 10:48:22 -0800 Subject: [PATCH 12/50] Plugged in image store in front-end (some commands are deactivated right now, will progressively reintroduce them --- container.go | 2 + dockerd/dockerd.go | 4 +- fs/changes.go | 144 +++++++++++++++++++++++ fs/store.go | 4 + server/server.go | 283 +++++++++++++++++++++++---------------------- 5 files changed, 295 insertions(+), 142 deletions(-) create mode 100644 fs/changes.go diff --git a/container.go b/container.go index ca7ab37f2..3f727726d 100644 --- a/container.go +++ b/container.go @@ -35,6 +35,7 @@ type Container struct { Config *Config Mountpoint *fs.Mountpoint State *State + Image string SysInitPath string lxcConfigPath string @@ -68,6 +69,7 @@ func createContainer(id string, root string, command string, args []string, imag Path: command, Args: args, Config: config, + Image: image.Id, Mountpoint: mountpoint, State: newState(), diff --git a/dockerd/dockerd.go b/dockerd/dockerd.go index b2337d342..7fff1eb7d 100644 --- a/dockerd/dockerd.go +++ b/dockerd/dockerd.go @@ -2,8 +2,8 @@ package main import ( "flag" - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/server" + ".." + "../server" "log" ) diff --git a/fs/changes.go b/fs/changes.go new file mode 100644 index 000000000..5c3523f3f --- /dev/null +++ b/fs/changes.go @@ -0,0 +1,144 @@ +package fs + +import ( + "fmt" + "path/filepath" + "os" + "strings" +) + +type ChangeType int + +const ( + ChangeModify = iota + ChangeAdd + ChangeDelete +) + +type Change struct { + Path string + Kind ChangeType +} + +func (change *Change) String() string { + var kind string + switch change.Kind { + case ChangeModify: + kind = "C" + case ChangeAdd: + kind = "A" + case ChangeDelete: + kind = "D" + } + return fmt.Sprintf("%s %s", kind, change.Path) +} + +func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { + var changes []Change + image, err := store.Get(mp.Image) + if err != nil { + return nil, err + } + layers, err := image.layers() + if err != nil { + return nil, err + } + + err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(mp.Rw, path) + if err != nil { + return err + } + path = filepath.Join("/", path) + + // Skip root + if path == "/" { + return nil + } + + // Skip AUFS metadata + if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { + return err + } + + change := Change{ + Path: path, + } + + // Find out what kind of modification happened + file := filepath.Base(path) + // If there is a whiteout, then the file was removed + if strings.HasPrefix(file, ".wh.") { + originalFile := strings.TrimLeft(file, ".wh.") + change.Path = filepath.Join(filepath.Dir(path), originalFile) + change.Kind = ChangeDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeAdd + + // ...Unless it already existed in a top layer, in which case, it's a modification + for _, layer := range layers { + stat, err := os.Stat(filepath.Join(layer, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the top layer, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeModify + break + } + } + } + + // Record change + changes = append(changes, change) + return nil + }) + if err != nil { + return nil, err + } + return changes, nil +} + +// Reset removes all changes to the filesystem, reverting it to its initial state. +func (mp *Mountpoint) Reset() error { + if err := os.RemoveAll(mp.Rw); err != nil { + return err + } + // We removed the RW directory itself along with its content: let's re-create an empty one. + if err := mp.createFolders(); err != nil { + return err + } + return nil +} + +// Open opens the named file for reading. +// func (fs *Filesystem) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { +// if err := fs.EnsureMounted(); err != nil { +// return nil, err +// } +// return os.OpenFile(filepath.Join(fs.RootFS, path), flag, perm) +// } + +// ReadDir reads the directory named by dirname, relative to the Filesystem's root, +// and returns a list of sorted directory entries +// func (fs *Filesystem) ReadDir(dirname string) ([]os.FileInfo, error) { +// if err := fs.EnsureMounted(); err != nil { +// return nil, err +// } +// return ioutil.ReadDir(filepath.Join(fs.RootFS, dirname)) +// } \ No newline at end of file diff --git a/fs/store.go b/fs/store.go index 9dd30e450..67b0f2b3b 100644 --- a/fs/store.go +++ b/fs/store.go @@ -157,6 +157,10 @@ func (store *Store) Register(image *Image, pth string) error { return trans.Commit() } +func (store *Store) Layers() []string { + return store.layers.List() +} + type Image struct { Id string Parent string diff --git a/server/server.go b/server/server.go index 8a3bb404f..e5df5ec5e 100644 --- a/server/server.go +++ b/server/server.go @@ -6,10 +6,10 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/image" - "github.com/dotcloud/docker/rcli" + ".." + "../future" + "../fs" + "../rcli" "io" "net/http" "net/url" @@ -68,7 +68,7 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, - len(srv.images.ById)) + 0) // FIXME: Number of images return nil } @@ -153,7 +153,7 @@ func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...stri } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Umount(); err != nil { + if err := container.Mountpoint.Umount(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -176,7 +176,7 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Mount(); err != nil { + if err := container.Mountpoint.EnsureMounted(); err != nil { return err } fmt.Fprintln(stdout, container.Id) @@ -187,73 +187,73 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Filesystem.OpenFile(path, os.O_RDONLY, 0); err != nil { - return err - } else if _, err := io.Copy(stdout, f); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} +// func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 2 { +// cmd.Usage() +// return nil +// } +// name, path := cmd.Arg(0), cmd.Arg(1) +// if container := srv.containers.Get(name); container != nil { +// if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { +// return err +// } else if _, err := io.Copy(stdout, f); err != nil { +// return err +// } +// return nil +// } +// return errors.New("No such container: " + name) +// } -func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if f, err := container.Filesystem.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { - return err - } else if _, err := io.Copy(f, stdin); err != nil { - return err - } - return nil - } - return errors.New("No such container: " + name) -} +// func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 2 { +// cmd.Usage() +// return nil +// } +// name, path := cmd.Arg(0), cmd.Arg(1) +// if container := srv.containers.Get(name); container != nil { +// if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { +// return err +// } else if _, err := io.Copy(f, stdin); err != nil { +// return err +// } +// return nil +// } +// return errors.New("No such container: " + name) +// } -func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 2 { - cmd.Usage() - return nil - } - name, path := cmd.Arg(0), cmd.Arg(1) - if container := srv.containers.Get(name); container != nil { - if files, err := container.Filesystem.ReadDir(path); err != nil { - return err - } else { - for _, f := range files { - fmt.Fprintln(stdout, f.Name()) - } - } - return nil - } - return errors.New("No such container: " + name) -} +// func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 2 { +// cmd.Usage() +// return nil +// } +// name, path := cmd.Arg(0), cmd.Arg(1) +// if container := srv.containers.Get(name); container != nil { +// if files, err := container.Mountpoint.ReadDir(path); err != nil { +// return err +// } else { +// for _, f := range files { +// fmt.Fprintln(stdout, f.Name()) +// } +// } +// return nil +// } +// return errors.New("No such container: " + name) +// } func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") @@ -269,8 +269,8 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.containers.Get(name); container != nil { obj = container - } else if image := srv.images.Find(name); image != nil { - obj = image + //} else if image, err := srv.images.List(name); image != nil { + // obj = image } else { return errors.New("No such container or image: " + name) } @@ -289,27 +289,27 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str } // 'docker rmi NAME' removes all images with the name NAME -func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") - if err := cmd.Parse(args); err != nil { - cmd.Usage() - return nil - } - if cmd.NArg() < 1 { - cmd.Usage() - return nil - } - for _, name := range cmd.Args() { - image := srv.images.Find(name) - if image == nil { - return errors.New("No such image: " + name) - } - if err := srv.images.Delete(name); err != nil { - return err - } - } - return nil -} +// func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { +// cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") +// if err := cmd.Parse(args); err != nil { +// cmd.Usage() +// return nil +// } +// if cmd.NArg() < 1 { +// cmd.Usage() +// return nil +// } +// for _, name := range cmd.Args() { +// image := srv.images.Find(name) +// if image == nil { +// return errors.New("No such image: " + name) +// } +// if err := srv.images.Delete(name); err != nil { +// return err +// } +// } +// return nil +// } func (srv *Server) CmdRm(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rm", "[OPTIONS] CONTAINER", "Remove a container") @@ -348,17 +348,9 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] NAME", "Download a new image from a remote location") - fl_bzip2 := cmd.Bool("j", false, "Bzip2 compression") - fl_gzip := cmd.Bool("z", false, "Gzip compression") if err := cmd.Parse(args); err != nil { return nil } - var compression image.Compression - if *fl_bzip2 { - compression = image.Bzip2 - } else if *fl_gzip { - compression = image.Gzip - } name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") @@ -380,7 +372,7 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string if err != nil { return err } - img, err := srv.images.Import(name, resp.Body, stdout, nil, compression) + img, err := srv.images.Create(resp.Body, nil, name, "") if err != nil { return err } @@ -390,22 +382,14 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdPut(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "put", "[OPTIONS] NAME", "Import a new image from a local archive.") - fl_bzip2 := cmd.Bool("j", false, "Bzip2 compression") - fl_gzip := cmd.Bool("z", false, "Gzip compression") if err := cmd.Parse(args); err != nil { return nil } - var compression image.Compression - if *fl_bzip2 { - compression = image.Bzip2 - } else if *fl_gzip { - compression = image.Gzip - } name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") } - img, err := srv.images.Import(name, stdin, stdout, nil, compression) + img, err := srv.images.Create(stdin, nil, name, "") if err != nil { return err } @@ -430,23 +414,27 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri if !*quiet { fmt.Fprintf(w, "NAME\tID\tCREATED\tPARENT\n") } - for _, name := range srv.images.Names() { + paths, err := srv.images.Paths() + if err != nil { + return err + } + for _, name := range paths { if nameFilter != "" && nameFilter != name { continue } - for idx, img := range *srv.images.ByName[name] { + ids, err := srv.images.List(name) + if err != nil { + return err + } + for idx, img := range ids { if *limit > 0 && idx >= *limit { break } if !*quiet { - id := img.Id - if !img.IdIsFinal() { - id += "..." - } for idx, field := range []string{ /* NAME */ name, - /* ID */ id, - /* CREATED */ future.HumanDuration(time.Now().Sub(img.Created)) + " ago", + /* ID */ img.Id, + /* CREATED */ future.HumanDuration(time.Now().Sub(time.Now())) + " ago", // FIXME: should be img.Created /* PARENT */ img.Parent, } { if idx == 0 { @@ -523,7 +511,7 @@ func (srv *Server) CmdLayers(stdin io.ReadCloser, stdout io.Writer, args ...stri if err := cmd.Parse(args); err != nil { return nil } - for _, layer := range srv.images.Layers.List() { + for _, layer := range srv.images.Layers() { fmt.Fprintln(stdout, layer) } return nil @@ -536,10 +524,16 @@ func (srv *Server) CmdCp(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := cmd.Parse(args); err != nil { return nil } - if newImage, err := srv.images.Copy(cmd.Arg(0), cmd.Arg(1)); err != nil { + if image, err := srv.images.Get(cmd.Arg(0)); err != nil { return err + } else if image == nil { + return errors.New("Image " + cmd.Arg(0) + " does not exist") } else { - fmt.Fprintln(stdout, newImage.Id) + if img, err := image.Copy(cmd.Arg(1)); err != nil { + return err + } else { + fmt.Fprintln(stdout, img.Id) + } } return nil } @@ -558,16 +552,21 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } if container := srv.containers.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := docker.Tar(container.Filesystem.RWPath) + rwTar, err := docker.Tar(container.Mountpoint.Rw) if err != nil { return err } // Create a new image from the container's base layers + a new layer from container changes - parentImg := srv.images.Find(container.GetUserData("image")) - img, err := srv.images.Import(imgName, rwTar, stdout, parentImg, image.Uncompressed) + parentImg, err := srv.images.Get(container.Image) if err != nil { return err } + + img, err := srv.images.Create(rwTar, parentImg, imgName, "") + if err != nil { + return err + } + fmt.Fprintln(stdout, img.Id) return nil } @@ -587,7 +586,10 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) } name := cmd.Arg(0) if container := srv.containers.Get(name); container != nil { - data, err := container.Filesystem.Tar() + if err := container.Mountpoint.EnsureMounted(); err != nil { + return err + } + data, err := docker.Tar(container.Mountpoint.Root) if err != nil { return err } @@ -613,7 +615,7 @@ func (srv *Server) CmdDiff(stdin io.ReadCloser, stdout io.Writer, args ...string if container := srv.containers.Get(cmd.Arg(0)); container == nil { return errors.New("No such container") } else { - changes, err := container.Filesystem.Changes() + changes, err := srv.images.Changes(container.Mountpoint) if err != nil { return err } @@ -636,7 +638,7 @@ func (srv *Server) CmdReset(stdin io.ReadCloser, stdout io.Writer, args ...strin } for _, name := range cmd.Args() { if container := srv.containers.Get(name); container != nil { - if err := container.Filesystem.Reset(); err != nil { + if err := container.Mountpoint.Reset(); err != nil { return errors.New("Reset " + container.Id + ": " + err.Error()) } } @@ -666,9 +668,9 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *image.Image, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *fs.Image, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] - container, err := srv.containers.Create(id, cmd, args, img.Layers, + container, err := srv.containers.Create(id, cmd, args, img, &docker.Config{Hostname: id, User: user, Tty: tty, OpenStdin: openStdin}) if err != nil { return nil, err @@ -757,8 +759,10 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) cmdline = []string{"/bin/bash", "-i"} } // Find the image - img := srv.images.Find(name) - if img == nil { + img, err := srv.images.Get(name) + if err != nil { + return err + } else if img == nil { return errors.New("No such image: " + name) } // Create new container @@ -820,16 +824,15 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) func New() (*Server, error) { future.Seed() - images, err := image.New("/var/lib/docker/images") - if err != nil { - return nil, err - } + // if err != nil { + // return nil, err + // } containers, err := docker.New() if err != nil { return nil, err } srv := &Server{ - images: images, + images: containers.Store, containers: containers, } return srv, nil @@ -876,5 +879,5 @@ func (srv *Server) CmdWeb(stdin io.ReadCloser, stdout io.Writer, args ...string) type Server struct { containers *docker.Docker - images *image.Store + images *fs.Store } From c59fff422fdc61c1c994b44fd83e4173a2f0530a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 9 Mar 2013 19:44:09 -0800 Subject: [PATCH 13/50] gofmt --- client/client.go | 4 +- client/term.go | 172 ++++++++++++++++++++-------------------- container.go | 6 +- docker.go | 12 +-- docker/docker.go | 3 +- docker_test.go | 8 +- dockerd/dockerd.go | 2 +- fake/fake.go | 16 ++-- fs/changes.go | 194 ++++++++++++++++++++++----------------------- fs/layers.go | 16 ++-- fs/layers_test.go | 9 +-- fs/mount_linux.go | 1 - fs/store.go | 5 +- fs/store_test.go | 2 +- future/future.go | 25 +++--- image/image.go | 57 +++++++------ mount_linux.go | 1 - rcli/http.go | 9 +-- rcli/tcp.go | 12 ++- rcli/types.go | 10 +-- server/server.go | 12 +-- 21 files changed, 273 insertions(+), 303 deletions(-) diff --git a/client/client.go b/client/client.go index 5a8aac380..4c4ea1c5e 100644 --- a/client/client.go +++ b/client/client.go @@ -1,8 +1,8 @@ package client import ( - "github.com/dotcloud/docker/rcli" "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "io" "io/ioutil" "log" @@ -112,7 +112,7 @@ func InteractiveMode(scripts ...string) error { return err } io.WriteString(rcfile, "enable -n help\n") - os.Setenv("PATH", tmp + ":" + os.Getenv("PATH")) + os.Setenv("PATH", tmp+":"+os.Getenv("PATH")) os.Setenv("PS1", "\\h docker> ") shell := exec.Command("/bin/bash", append([]string{"--rcfile", rcfile.Name()}, scripts...)...) shell.Stdin = os.Stdin diff --git a/client/term.go b/client/term.go index ed52be96b..a988d0d79 100644 --- a/client/term.go +++ b/client/term.go @@ -15,7 +15,6 @@ type Termios struct { Ospeed uintptr } - const ( // Input flags inpck = 0x010 @@ -35,113 +34,110 @@ const ( ) const ( - HUPCL = 0x4000 - ICANON = 0x100 - ICRNL = 0x100 - IEXTEN = 0x400 - BRKINT = 0x2 - CFLUSH = 0xf - CLOCAL = 0x8000 - CREAD = 0x800 - CS5 = 0x0 - CS6 = 0x100 - CS7 = 0x200 - CS8 = 0x300 - CSIZE = 0x300 - CSTART = 0x11 - CSTATUS = 0x14 - CSTOP = 0x13 - CSTOPB = 0x400 - CSUSP = 0x1a - IGNBRK = 0x1 - IGNCR = 0x80 - IGNPAR = 0x4 - IMAXBEL = 0x2000 - INLCR = 0x40 - INPCK = 0x10 - ISIG = 0x80 - ISTRIP = 0x20 - IUTF8 = 0x4000 - IXANY = 0x800 - IXOFF = 0x400 - IXON = 0x200 - NOFLSH = 0x80000000 - OCRNL = 0x10 - OFDEL = 0x20000 - OFILL = 0x80 - ONLCR = 0x2 - ONLRET = 0x40 - ONOCR = 0x20 - ONOEOT = 0x8 - OPOST = 0x1 -RENB = 0x1000 - PARMRK = 0x8 - PARODD = 0x2000 + HUPCL = 0x4000 + ICANON = 0x100 + ICRNL = 0x100 + IEXTEN = 0x400 + BRKINT = 0x2 + CFLUSH = 0xf + CLOCAL = 0x8000 + CREAD = 0x800 + CS5 = 0x0 + CS6 = 0x100 + CS7 = 0x200 + CS8 = 0x300 + CSIZE = 0x300 + CSTART = 0x11 + CSTATUS = 0x14 + CSTOP = 0x13 + CSTOPB = 0x400 + CSUSP = 0x1a + IGNBRK = 0x1 + IGNCR = 0x80 + IGNPAR = 0x4 + IMAXBEL = 0x2000 + INLCR = 0x40 + INPCK = 0x10 + ISIG = 0x80 + ISTRIP = 0x20 + IUTF8 = 0x4000 + IXANY = 0x800 + IXOFF = 0x400 + IXON = 0x200 + NOFLSH = 0x80000000 + OCRNL = 0x10 + OFDEL = 0x20000 + OFILL = 0x80 + ONLCR = 0x2 + ONLRET = 0x40 + ONOCR = 0x20 + ONOEOT = 0x8 + OPOST = 0x1 + RENB = 0x1000 + PARMRK = 0x8 + PARODD = 0x2000 - TOSTOP = 0x400000 - VDISCARD = 0xf - VDSUSP = 0xb - VEOF = 0x0 - VEOL = 0x1 - VEOL2 = 0x2 - VERASE = 0x3 - VINTR = 0x8 - VKILL = 0x5 - VLNEXT = 0xe - VMIN = 0x10 - VQUIT = 0x9 - VREPRINT = 0x6 - VSTART = 0xc - VSTATUS = 0x12 - VSTOP = 0xd - VSUSP = 0xa - VT0 = 0x0 - VT1 = 0x10000 - VTDLY = 0x10000 - VTIME = 0x11 - ECHO = 0x00000008 + TOSTOP = 0x400000 + VDISCARD = 0xf + VDSUSP = 0xb + VEOF = 0x0 + VEOL = 0x1 + VEOL2 = 0x2 + VERASE = 0x3 + VINTR = 0x8 + VKILL = 0x5 + VLNEXT = 0xe + VMIN = 0x10 + VQUIT = 0x9 + VREPRINT = 0x6 + VSTART = 0xc + VSTATUS = 0x12 + VSTOP = 0xd + VSUSP = 0xa + VT0 = 0x0 + VT1 = 0x10000 + VTDLY = 0x10000 + VTIME = 0x11 + ECHO = 0x00000008 - PENDIN = 0x20000000 + PENDIN = 0x20000000 ) type State struct { - termios Termios + termios Termios } // IsTerminal returns true if the given file descriptor is a terminal. func IsTerminal(fd int) bool { - var termios Termios - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&termios)), 0, 0, 0) - return err == 0 + var termios Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 } // MakeRaw put the terminal connected to the given file descriptor into raw // mode and returns the previous state of the terminal so that it can be // restored. func MakeRaw(fd int) (*State, error) { - var oldState State - if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 { - return nil, err - } + var oldState State + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 { + return nil, err + } - newState := oldState.termios - newState.Iflag &^= ISTRIP | INLCR | IGNCR | IXON | IXOFF - newState.Iflag |= ICRNL - newState.Oflag |= ONLCR - newState.Lflag &^= ECHO | ICANON | ISIG - if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { - return nil, err - } + newState := oldState.termios + newState.Iflag &^= ISTRIP | INLCR | IGNCR | IXON | IXOFF + newState.Iflag |= ICRNL + newState.Oflag |= ONLCR + newState.Lflag &^= ECHO | ICANON | ISIG + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { + return nil, err + } - return &oldState, nil + return &oldState, nil } - // Restore restores the terminal connected to the given file descriptor to a // previous state. func Restore(fd int, state *State) error { - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) - return err + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) + return err } - - diff --git a/container.go b/container.go index 3f727726d..5a4559f72 100644 --- a/container.go +++ b/container.go @@ -1,6 +1,7 @@ package docker import ( + "./fs" "bytes" "encoding/json" "errors" @@ -14,7 +15,6 @@ import ( "strings" "syscall" "time" - "./fs" ) var sysInitPath string @@ -35,7 +35,7 @@ type Container struct { Config *Config Mountpoint *fs.Mountpoint State *State - Image string + Image string SysInitPath string lxcConfigPath string @@ -69,7 +69,7 @@ func createContainer(id string, root string, command string, args []string, imag Path: command, Args: args, Config: config, - Image: image.Id, + Image: image.Id, Mountpoint: mountpoint, State: newState(), diff --git a/docker.go b/docker.go index b5ee2bb84..105b1ea7d 100644 --- a/docker.go +++ b/docker.go @@ -1,6 +1,7 @@ package docker import ( + "./fs" "container/list" "fmt" "io/ioutil" @@ -8,14 +9,13 @@ import ( "os" "path" "sort" - "./fs" ) type Docker struct { - root string - repository string - containers *list.List - Store *fs.Store + root string + repository string + containers *list.List + Store *fs.Store } func (docker *Docker) List() []*Container { @@ -117,7 +117,7 @@ func NewFromDirectory(root string) (*Docker, error) { root: root, repository: path.Join(root, "containers"), containers: list.New(), - Store: store, + Store: store, } if err := os.MkdirAll(docker.repository, 0700); err != nil && !os.IsExist(err) { diff --git a/docker/docker.go b/docker/docker.go index efc93620a..fa9011def 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -2,10 +2,10 @@ package main import ( "flag" + "github.com/dotcloud/docker/client" "log" "os" "path" - "github.com/dotcloud/docker/client" ) func main() { @@ -27,4 +27,3 @@ func main() { } } } - diff --git a/docker_test.go b/docker_test.go index c612c4f60..befc0a9a3 100644 --- a/docker_test.go +++ b/docker_test.go @@ -1,17 +1,17 @@ package docker import ( + "./fs" + "io" "io/ioutil" "log" "os" "testing" - "io" - "./fs" ) const testLayerPath string = "/var/lib/docker/docker-ut.tar" -func layerArchive(tarfile string) (io.Reader, error) { +func layerArchive(tarfile string) (io.Reader, error) { // FIXME: need to close f somewhere f, err := os.Open(tarfile) if err != nil { @@ -57,7 +57,7 @@ func newTestDocker() (*Docker, error) { return docker, nil } -func GetTestImage(docker *Docker) (*fs.Image) { +func GetTestImage(docker *Docker) *fs.Image { imgs, err := docker.Store.Images() if err != nil { panic(err) diff --git a/dockerd/dockerd.go b/dockerd/dockerd.go index 7fff1eb7d..22ae8df8e 100644 --- a/dockerd/dockerd.go +++ b/dockerd/dockerd.go @@ -1,9 +1,9 @@ package main import ( - "flag" ".." "../server" + "flag" "log" ) diff --git a/fake/fake.go b/fake/fake.go index b3ec46310..598ff6a81 100644 --- a/fake/fake.go +++ b/fake/fake.go @@ -1,20 +1,19 @@ package fake import ( - "bytes" - "math/rand" - "io" "archive/tar" - "os/exec" + "bytes" "github.com/kr/pty" + "io" + "math/rand" + "os/exec" ) - func FakeTar() (io.Reader, error) { content := []byte("Hello world!\n") buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - for _, name := range []string {"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { + for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} { hdr := new(tar.Header) hdr.Size = int64(len(content)) hdr.Name = name @@ -27,7 +26,6 @@ func FakeTar() (io.Reader, error) { return buf, nil } - func WriteFakeTar(dst io.Writer) error { if data, err := FakeTar(); err != nil { return err @@ -37,7 +35,6 @@ func WriteFakeTar(dst io.Writer) error { return nil } - func RandomBytesChanged() uint { return uint(rand.Int31n(24 * 1024 * 1024)) } @@ -54,7 +51,6 @@ func ContainerRunning() bool { return false } - func StartCommand(cmd *exec.Cmd, interactive bool) (io.WriteCloser, io.ReadCloser, error) { if interactive { term, err := pty.Start(cmd) @@ -76,5 +72,3 @@ func StartCommand(cmd *exec.Cmd, interactive bool) (io.WriteCloser, io.ReadClose } return stdin, stdout, nil } - - diff --git a/fs/changes.go b/fs/changes.go index 5c3523f3f..9bd315ad8 100644 --- a/fs/changes.go +++ b/fs/changes.go @@ -1,129 +1,129 @@ package fs import ( - "fmt" - "path/filepath" - "os" - "strings" + "fmt" + "os" + "path/filepath" + "strings" ) type ChangeType int const ( - ChangeModify = iota - ChangeAdd - ChangeDelete + ChangeModify = iota + ChangeAdd + ChangeDelete ) type Change struct { - Path string - Kind ChangeType + Path string + Kind ChangeType } func (change *Change) String() string { - var kind string - switch change.Kind { - case ChangeModify: - kind = "C" - case ChangeAdd: - kind = "A" - case ChangeDelete: - kind = "D" - } - return fmt.Sprintf("%s %s", kind, change.Path) + var kind string + switch change.Kind { + case ChangeModify: + kind = "C" + case ChangeAdd: + kind = "A" + case ChangeDelete: + kind = "D" + } + return fmt.Sprintf("%s %s", kind, change.Path) } func (store *Store) Changes(mp *Mountpoint) ([]Change, error) { - var changes []Change - image, err := store.Get(mp.Image) - if err != nil { - return nil, err - } - layers, err := image.layers() - if err != nil { - return nil, err - } + var changes []Change + image, err := store.Get(mp.Image) + if err != nil { + return nil, err + } + layers, err := image.layers() + if err != nil { + return nil, err + } - err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } + err = filepath.Walk(mp.Rw, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } - // Rebase path - path, err = filepath.Rel(mp.Rw, path) - if err != nil { - return err - } - path = filepath.Join("/", path) + // Rebase path + path, err = filepath.Rel(mp.Rw, path) + if err != nil { + return err + } + path = filepath.Join("/", path) - // Skip root - if path == "/" { - return nil - } + // Skip root + if path == "/" { + return nil + } - // Skip AUFS metadata - if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { - return err - } + // Skip AUFS metadata + if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched { + return err + } - change := Change{ - Path: path, - } + change := Change{ + Path: path, + } - // Find out what kind of modification happened - file := filepath.Base(path) - // If there is a whiteout, then the file was removed - if strings.HasPrefix(file, ".wh.") { - originalFile := strings.TrimLeft(file, ".wh.") - change.Path = filepath.Join(filepath.Dir(path), originalFile) - change.Kind = ChangeDelete - } else { - // Otherwise, the file was added - change.Kind = ChangeAdd + // Find out what kind of modification happened + file := filepath.Base(path) + // If there is a whiteout, then the file was removed + if strings.HasPrefix(file, ".wh.") { + originalFile := strings.TrimLeft(file, ".wh.") + change.Path = filepath.Join(filepath.Dir(path), originalFile) + change.Kind = ChangeDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeAdd - // ...Unless it already existed in a top layer, in which case, it's a modification - for _, layer := range layers { - stat, err := os.Stat(filepath.Join(layer, path)) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - // The file existed in the top layer, so that's a modification + // ...Unless it already existed in a top layer, in which case, it's a modification + for _, layer := range layers { + stat, err := os.Stat(filepath.Join(layer, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the top layer, so that's a modification - // However, if it's a directory, maybe it wasn't actually modified. - // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar - if stat.IsDir() && f.IsDir() { - if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { - // Both directories are the same, don't record the change - return nil - } - } - change.Kind = ChangeModify - break - } - } - } + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeModify + break + } + } + } - // Record change - changes = append(changes, change) - return nil - }) - if err != nil { - return nil, err - } - return changes, nil + // Record change + changes = append(changes, change) + return nil + }) + if err != nil { + return nil, err + } + return changes, nil } // Reset removes all changes to the filesystem, reverting it to its initial state. func (mp *Mountpoint) Reset() error { - if err := os.RemoveAll(mp.Rw); err != nil { - return err - } - // We removed the RW directory itself along with its content: let's re-create an empty one. - if err := mp.createFolders(); err != nil { - return err - } - return nil + if err := os.RemoveAll(mp.Rw); err != nil { + return err + } + // We removed the RW directory itself along with its content: let's re-create an empty one. + if err := mp.createFolders(); err != nil { + return err + } + return nil } // Open opens the named file for reading. @@ -141,4 +141,4 @@ func (mp *Mountpoint) Reset() error { // return nil, err // } // return ioutil.ReadDir(filepath.Join(fs.RootFS, dirname)) -// } \ No newline at end of file +// } diff --git a/fs/layers.go b/fs/layers.go index bbb0ee201..d094ba569 100644 --- a/fs/layers.go +++ b/fs/layers.go @@ -1,25 +1,25 @@ package fs import ( + "../future" "errors" - "path" - "path/filepath" + "fmt" "io" "io/ioutil" "os" "os/exec" - "fmt" - "../future" + "path" + "path/filepath" ) type LayerStore struct { - Root string + Root string } type Compression uint32 const ( - Uncompressed Compression = iota + Uncompressed Compression = iota Bzip2 Gzip ) @@ -80,10 +80,9 @@ func (store *LayerStore) Init() error { return os.Mkdir(store.Root, 0700) } - func (store *LayerStore) Mktemp() (string, error) { tmpName := future.RandomId() - tmpPath := path.Join(store.Root, "tmp-" + tmpName) + tmpPath := path.Join(store.Root, "tmp-"+tmpName) if err := os.Mkdir(tmpPath, 0700); err != nil { return "", err } @@ -94,7 +93,6 @@ func (store *LayerStore) layerPath(id string) string { return path.Join(store.Root, id) } - func (store *LayerStore) AddLayer(id string, archive Archive, stderr io.Writer, compression Compression) (string, error) { if _, err := os.Stat(store.layerPath(id)); err == nil { return "", errors.New("Layer already exists: " + id) diff --git a/fs/layers_test.go b/fs/layers_test.go index 3d8e9e32a..8d47dc4b9 100644 --- a/fs/layers_test.go +++ b/fs/layers_test.go @@ -1,14 +1,12 @@ package fs import ( - "io/ioutil" - "testing" - "os" "github.com/dotcloud/docker/fake" + "io/ioutil" + "os" + "testing" ) - - func TestLayersInit(t *testing.T) { store := tempStore(t) defer os.RemoveAll(store.Root) @@ -54,7 +52,6 @@ func TestAddLayerDuplicate(t *testing.T) { } } - /* * HELPER FUNCTIONS */ diff --git a/fs/mount_linux.go b/fs/mount_linux.go index 6dd2a39af..b36888f75 100644 --- a/fs/mount_linux.go +++ b/fs/mount_linux.go @@ -2,7 +2,6 @@ package fs import "syscall" - func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { return syscall.Mount(source, target, fstype, flags, data) } diff --git a/fs/store.go b/fs/store.go index 67b0f2b3b..697919a35 100644 --- a/fs/store.go +++ b/fs/store.go @@ -10,9 +10,9 @@ import ( "io" "os" "path" + "path/filepath" "syscall" "time" - "path/filepath" ) type Store struct { @@ -168,7 +168,6 @@ type Image struct { store *Store `db:"-"` } - func (image *Image) Copy(pth string) (*Image, error) { if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { return nil, err @@ -198,7 +197,7 @@ func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { func (image *Image) layers() ([]string, error) { var list []string - var err error + var err error currentImg := image for currentImg != nil { if layer := image.store.layers.Get(image.Id); layer != "" { diff --git a/fs/store_test.go b/fs/store_test.go index 954c83d86..d5715d1ba 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -1,9 +1,9 @@ package fs import ( + "../fake" "errors" "fmt" - "../fake" "io/ioutil" "os" "testing" diff --git a/future/future.go b/future/future.go index a0efacc03..cc7771cf9 100644 --- a/future/future.go +++ b/future/future.go @@ -1,12 +1,12 @@ package future import ( - "crypto/sha256" - "io" - "fmt" - "time" "bytes" + "crypto/sha256" + "fmt" + "io" "math/rand" + "time" ) func Seed() { @@ -30,18 +30,18 @@ func HumanDuration(d time.Duration) string { return "About a minute" } else if minutes < 60 { return fmt.Sprintf("%d minutes", minutes) - } else if hours := int(d.Hours()); hours == 1{ + } else if hours := int(d.Hours()); hours == 1 { return "About an hour" } else if hours < 48 { return fmt.Sprintf("%d hours", hours) - } else if hours < 24 * 7 * 2 { - return fmt.Sprintf("%d days", hours / 24) - } else if hours < 24 * 30 * 3 { - return fmt.Sprintf("%d weeks", hours / 24 / 7) - } else if hours < 24 * 365 * 2 { - return fmt.Sprintf("%d months", hours / 24 / 30) + } else if hours < 24*7*2 { + return fmt.Sprintf("%d days", hours/24) + } else if hours < 24*30*3 { + return fmt.Sprintf("%d weeks", hours/24/7) + } else if hours < 24*365*2 { + return fmt.Sprintf("%d months", hours/24/30) } - return fmt.Sprintf("%d years", d.Hours() / 24 / 365) + return fmt.Sprintf("%d years", d.Hours()/24/365) } func randomBytes() io.Reader { @@ -60,4 +60,3 @@ func Go(f func() error) chan error { }() return ch } - diff --git a/image/image.go b/image/image.go index 82c88b2a0..5e7d27425 100644 --- a/image/image.go +++ b/image/image.go @@ -1,27 +1,25 @@ package image import ( + "encoding/json" + "errors" + "github.com/dotcloud/docker/future" "io" "io/ioutil" - "encoding/json" - "time" + "os" "path" "path/filepath" - "errors" "sort" - "os" - "github.com/dotcloud/docker/future" "strings" + "time" ) - type Store struct { *Index - Root string - Layers *LayerStore + Root string + Layers *LayerStore } - func New(root string) (*Store, error) { abspath, err := filepath.Abs(root) if err != nil { @@ -38,8 +36,8 @@ func New(root string) (*Store, error) { return nil, err } return &Store{ - Root: abspath, - Index: NewIndex(path.Join(root, "index.json")), + Root: abspath, + Index: NewIndex(path.Join(root, "index.json")), Layers: layers, }, nil } @@ -47,7 +45,7 @@ func New(root string) (*Store, error) { type Compression uint32 const ( - Uncompressed Compression = iota + Uncompressed Compression = iota Bzip2 Gzip ) @@ -79,20 +77,19 @@ func (store *Store) Create(name string, source string, layers ...string) (*Image return image, nil } - // Index type Index struct { - Path string - ByName map[string]*History - ById map[string]*Image + Path string + ByName map[string]*History + ById map[string]*Image } func NewIndex(path string) *Index { return &Index{ - Path: path, + Path: path, ByName: make(map[string]*History), - ById: make(map[string]*Image), + ById: make(map[string]*Image), } } @@ -222,7 +219,7 @@ func (index *Index) Names() []string { if err := index.load(); err != nil { return []string{} } - var names[]string + var names []string for name := range index.ByName { names = append(names, name) } @@ -285,23 +282,23 @@ func (history *History) Add(image *Image) { func (history *History) Del(id string) { for idx, image := range *history { if image.Id == id { - *history = append((*history)[:idx], (*history)[idx + 1:]...) + *history = append((*history)[:idx], (*history)[idx+1:]...) } } } type Image struct { - Id string // Globally unique identifier - Layers []string // Absolute paths - Created time.Time - Parent string + Id string // Globally unique identifier + Layers []string // Absolute paths + Created time.Time + Parent string } func (image *Image) IdParts() (string, string) { if len(image.Id) < 8 { return "", image.Id } - hash := image.Id[len(image.Id)-8:len(image.Id)] + hash := image.Id[len(image.Id)-8 : len(image.Id)] name := image.Id[:len(image.Id)-9] return name, hash } @@ -322,7 +319,7 @@ func generateImageId(name string, layers []string) (string, error) { for _, layer := range layers { ids += path.Base(layer) } - if h, err := future.ComputeId(strings.NewReader(ids)); err != nil { + if h, err := future.ComputeId(strings.NewReader(ids)); err != nil { return "", err } else { hash = h @@ -337,9 +334,9 @@ func NewImage(name string, layers []string, parent string) (*Image, error) { return nil, err } return &Image{ - Id: id, - Layers: layers, - Created: time.Now(), - Parent: parent, + Id: id, + Layers: layers, + Created: time.Now(), + Parent: parent, }, nil } diff --git a/mount_linux.go b/mount_linux.go index a5a24e848..0efb25300 100644 --- a/mount_linux.go +++ b/mount_linux.go @@ -2,7 +2,6 @@ package docker import "syscall" - func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { return syscall.Mount(source, target, fstype, flags, data) } diff --git a/rcli/http.go b/rcli/http.go index e6cb5657d..cc8d3b149 100644 --- a/rcli/http.go +++ b/rcli/http.go @@ -1,13 +1,12 @@ package rcli import ( + "fmt" "net/http" "net/url" "path" - "fmt" ) - // Use this key to encode an RPC call into an URL, // eg. domain.tld/path/to/method?q=get_user&q=gordon const ARG_URL_KEY = "q" @@ -16,18 +15,16 @@ func URLToCall(u *url.URL) (method string, args []string) { return path.Base(u.Path), u.Query()[ARG_URL_KEY] } - func ListenAndServeHTTP(addr string, service Service) error { return http.ListenAndServe(addr, http.HandlerFunc( - func (w http.ResponseWriter, r *http.Request) { + func(w http.ResponseWriter, r *http.Request) { cmd, args := URLToCall(r.URL) if err := call(service, r.Body, &AutoFlush{w}, append([]string{cmd}, args...)...); err != nil { - fmt.Fprintf(w, "Error: " + err.Error() + "\n") + fmt.Fprintf(w, "Error: "+err.Error()+"\n") } })) } - type AutoFlush struct { http.ResponseWriter } diff --git a/rcli/tcp.go b/rcli/tcp.go index 0a06d459c..869a3bcdb 100644 --- a/rcli/tcp.go +++ b/rcli/tcp.go @@ -1,13 +1,13 @@ package rcli import ( + "bufio" + "encoding/json" + "fmt" "io" "io/ioutil" - "net" "log" - "fmt" - "encoding/json" - "bufio" + "net" ) // Connect to a remote endpoint using protocol `proto` and address `addr`, @@ -44,7 +44,7 @@ func ListenAndServe(proto, addr string, service Service) error { go func() { if err := Serve(conn, service); err != nil { log.Printf("Error: " + err.Error() + "\n") - fmt.Fprintf(conn, "Error: " + err.Error() + "\n") + fmt.Fprintf(conn, "Error: "+err.Error()+"\n") } conn.Close() }() @@ -53,7 +53,6 @@ func ListenAndServe(proto, addr string, service Service) error { return nil } - // Parse an rcli call on a new connection, and pass it to `service` if it // is valid. func Serve(conn io.ReadWriter, service Service) error { @@ -68,4 +67,3 @@ func Serve(conn io.ReadWriter, service Service) error { } return nil } - diff --git a/rcli/types.go b/rcli/types.go index b8572cd89..52079291b 100644 --- a/rcli/types.go +++ b/rcli/types.go @@ -8,13 +8,13 @@ package rcli // are the usual suspects. import ( + "errors" + "flag" "fmt" "io" - "reflect" - "flag" "log" + "reflect" "strings" - "errors" ) type Service interface { @@ -25,7 +25,6 @@ type Service interface { type Cmd func(io.ReadCloser, io.Writer, ...string) error type CmdMethod func(Service, io.ReadCloser, io.Writer, ...string) error - func call(service Service, stdin io.ReadCloser, stdout io.Writer, args ...string) error { if len(args) == 0 { args = []string{"help"} @@ -63,7 +62,7 @@ func getMethod(service Service, name string) Cmd { return nil } } - methodName := "Cmd"+strings.ToUpper(name[:1])+strings.ToLower(name[1:]) + methodName := "Cmd" + strings.ToUpper(name[:1]) + strings.ToLower(name[1:]) method, exists := reflect.TypeOf(service).MethodByName(methodName) if !exists { return nil @@ -91,4 +90,3 @@ func Subcmd(output io.Writer, name, signature, description string) *flag.FlagSet } return flags } - diff --git a/server/server.go b/server/server.go index e5df5ec5e..5dffe0979 100644 --- a/server/server.go +++ b/server/server.go @@ -1,15 +1,15 @@ package server import ( + ".." + "../fs" + "../future" + "../rcli" "bufio" "bytes" "encoding/json" "errors" "fmt" - ".." - "../future" - "../fs" - "../rcli" "io" "net/http" "net/url" @@ -269,8 +269,8 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str var obj interface{} if container := srv.containers.Get(name); container != nil { obj = container - //} else if image, err := srv.images.List(name); image != nil { - // obj = image + //} else if image, err := srv.images.List(name); image != nil { + // obj = image } else { return errors.New("No such container or image: " + name) } From 4474cd5677751ee5acac0525953eea583f797a1b Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 9 Mar 2013 19:49:09 -0800 Subject: [PATCH 14/50] Fixed a bug which caused docker to crash at startup while restoring existing containers --- container.go | 10 +++++++++- docker.go | 3 +-- fs/store.go | 8 ++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/container.go b/container.go index 5a4559f72..58ece78f5 100644 --- a/container.go +++ b/container.go @@ -100,17 +100,25 @@ func createContainer(id string, root string, command string, args []string, imag return container, nil } -func loadContainer(containerPath string) (*Container, error) { +func loadContainer(store *fs.Store, containerPath string) (*Container, error) { data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) if err != nil { return nil, err } + mountpoint, err := store.FetchMountpoint( + path.Join(containerPath, "rootfs"), + path.Join(containerPath, "rw"), + ) + if err != nil { + return nil, err + } container := &Container{ stdout: newWriteBroadcaster(), stderr: newWriteBroadcaster(), stdoutLog: new(bytes.Buffer), stderrLog: new(bytes.Buffer), lxcConfigPath: path.Join(containerPath, "config.lxc"), + Mountpoint: mountpoint, } if err := json.Unmarshal(data, container); err != nil { return nil, err diff --git a/docker.go b/docker.go index 105b1ea7d..50b38dbed 100644 --- a/docker.go +++ b/docker.go @@ -92,12 +92,11 @@ func (docker *Docker) restore() error { return err } for _, v := range dir { - container, err := loadContainer(path.Join(docker.repository, v.Name())) + container, err := loadContainer(docker.Store, path.Join(docker.repository, v.Name())) if err != nil { log.Printf("Failed to load container %v: %v", v.Name(), err) continue } - container.Mountpoint.Store = docker.Store docker.containers.PushBack(container) } return nil diff --git a/fs/store.go b/fs/store.go index 697919a35..1ced3722f 100644 --- a/fs/store.go +++ b/fs/store.go @@ -227,7 +227,7 @@ func (image *Image) Mountpoints() ([]*Mountpoint, error) { func (image *Image) Mount(root, rw string) (*Mountpoint, error) { var mountpoint *Mountpoint - if mp, err := image.fetchMountpoint(root, rw); err != nil { + if mp, err := image.store.FetchMountpoint(root, rw); err != nil { return nil, err } else if mp == nil { mountpoint, err = image.Mountpoint(root, rw) @@ -345,8 +345,8 @@ func (mp *Mountpoint) Deregister() error { return err } -func (image *Image) fetchMountpoint(root, rw string) (*Mountpoint, error) { - res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=? and Root=? and Rw=?", image.Id, root, rw) +func (store *Store) FetchMountpoint(root, rw string) (*Mountpoint, error) { + res, err := store.orm.Select(Mountpoint{}, "select * from mountpoints where Root=? and Rw=?", root, rw) if err != nil { return nil, err } else if len(res) < 1 || res[0] == nil { @@ -354,7 +354,7 @@ func (image *Image) fetchMountpoint(root, rw string) (*Mountpoint, error) { } mp := res[0].(*Mountpoint) - mp.Store = image.store + mp.Store = store return mp, nil } From b5b2f005eb679d0224db407ad1e9320be361640a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 9 Mar 2013 19:56:33 -0800 Subject: [PATCH 15/50] Fixed a bug which caused dockerd to not create its DB if an empty /var/lib/docker/images existed; fixed a bug which caused dockerd to not create missing tables in the images DB. --- fs/store.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/store.go b/fs/store.go index 1ced3722f..ed70532ab 100644 --- a/fs/store.go +++ b/fs/store.go @@ -4,9 +4,9 @@ import ( "database/sql" "errors" "fmt" - "github.com/coopernurse/gorp" "github.com/dotcloud/docker/future" _ "github.com/mattn/go-sqlite3" + "github.com/shykes/gorp" //Forked to implement CreateTablesOpts "io" "os" "path" @@ -29,8 +29,6 @@ func New(root string) (*Store, error) { if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { return nil, err - } else if os.IsExist(err) { - isNewStore = false } db, err := sql.Open("sqlite3", path.Join(root, "db")) if err != nil { @@ -42,7 +40,7 @@ func New(root string) (*Store, error) { orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") if isNewStore { - if err := orm.CreateTables(); err != nil { + if err := orm.CreateTablesOpts(true); err != nil { return nil, err } } From dd0227b700503e03fce0e9e9ea3539c438e9c0cb Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 9 Mar 2013 20:17:31 -0800 Subject: [PATCH 16/50] Removed deprecated docker/image package --- image/image.go | 342 ------------------------------------------------- 1 file changed, 342 deletions(-) delete mode 100644 image/image.go diff --git a/image/image.go b/image/image.go deleted file mode 100644 index 5e7d27425..000000000 --- a/image/image.go +++ /dev/null @@ -1,342 +0,0 @@ -package image - -import ( - "encoding/json" - "errors" - "github.com/dotcloud/docker/future" - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "sort" - "strings" - "time" -) - -type Store struct { - *Index - Root string - Layers *LayerStore -} - -func New(root string) (*Store, error) { - abspath, err := filepath.Abs(root) - if err != nil { - return nil, err - } - if err := os.MkdirAll(abspath, 0700); err != nil && !os.IsExist(err) { - return nil, err - } - layers, err := NewLayerStore(path.Join(root, "layers")) - if err != nil { - return nil, err - } - if err := layers.Init(); err != nil { - return nil, err - } - return &Store{ - Root: abspath, - Index: NewIndex(path.Join(root, "index.json")), - Layers: layers, - }, nil -} - -type Compression uint32 - -const ( - Uncompressed Compression = iota - Bzip2 - Gzip -) - -func (store *Store) Import(name string, archive io.Reader, stderr io.Writer, parent *Image, compression Compression) (*Image, error) { - layer, err := store.Layers.AddLayer(archive, stderr, compression) - if err != nil { - return nil, err - } - layers := []string{layer} - if parent != nil { - layers = append(layers, parent.Layers...) - } - var parentId string - if parent != nil { - parentId = parent.Id - } - return store.Create(name, parentId, layers...) -} - -func (store *Store) Create(name string, source string, layers ...string) (*Image, error) { - image, err := NewImage(name, layers, source) - if err != nil { - return nil, err - } - if err := store.Index.Add(name, image); err != nil { - return nil, err - } - return image, nil -} - -// Index - -type Index struct { - Path string - ByName map[string]*History - ById map[string]*Image -} - -func NewIndex(path string) *Index { - return &Index{ - Path: path, - ByName: make(map[string]*History), - ById: make(map[string]*Image), - } -} - -func (index *Index) Exists(id string) bool { - _, exists := index.ById[id] - return exists -} - -func (index *Index) Find(idOrName string) *Image { - // Load - if err := index.load(); err != nil { - return nil - } - // Lookup by ID - if image, exists := index.ById[idOrName]; exists { - return image - } - // Lookup by name - if history, exists := index.ByName[idOrName]; exists && history.Len() > 0 { - return (*history)[0] - } - return nil -} - -func (index *Index) Add(name string, image *Image) error { - // Load - if err := index.load(); err != nil { - return err - } - if _, exists := index.ByName[name]; !exists { - index.ByName[name] = new(History) - } else { - // If this image is already the latest version, don't add it. - if (*index.ByName[name])[0].Id == image.Id { - return nil - } - } - index.ByName[name].Add(image) - index.ById[image.Id] = image - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -func (index *Index) Copy(srcNameOrId, dstName string) (*Image, error) { - if srcNameOrId == "" || dstName == "" { - return nil, errors.New("Illegal image name") - } - // Load - if err := index.load(); err != nil { - return nil, err - } - src := index.Find(srcNameOrId) - if src == nil { - return nil, errors.New("No such image: " + srcNameOrId) - } - dst, err := NewImage(dstName, src.Layers, src.Id) - if err != nil { - return nil, err - } - if err := index.Add(dstName, dst); err != nil { - return nil, err - } - // Save - if err := index.save(); err != nil { - return nil, err - } - return dst, nil -} - -func (index *Index) Rename(oldName, newName string) error { - // Load - if err := index.load(); err != nil { - return err - } - if _, exists := index.ByName[oldName]; !exists { - return errors.New("Can't rename " + oldName + ": no such image.") - } - if _, exists := index.ByName[newName]; exists { - return errors.New("Can't rename to " + newName + ": name is already in use.") - } - index.ByName[newName] = index.ByName[oldName] - delete(index.ByName, oldName) - // Change the ID of all images, since they include the name - for _, image := range *index.ByName[newName] { - if id, err := generateImageId(newName, image.Layers); err != nil { - return err - } else { - oldId := image.Id - image.Id = id - index.ById[id] = image - delete(index.ById, oldId) - } - } - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -// Delete deletes all images with the name `name` -func (index *Index) Delete(name string) error { - // Load - if err := index.load(); err != nil { - return err - } - if _, exists := index.ByName[name]; !exists { - return errors.New("No such image: " + name) - } - // Remove from index lookup - for _, image := range *index.ByName[name] { - delete(index.ById, image.Id) - } - // Remove from name lookup - delete(index.ByName, name) - // Save - if err := index.save(); err != nil { - return err - } - return nil -} - -func (index *Index) Names() []string { - if err := index.load(); err != nil { - return []string{} - } - var names []string - for name := range index.ByName { - names = append(names, name) - } - sort.Strings(names) - return names -} - -func (index *Index) load() error { - jsonData, err := ioutil.ReadFile(index.Path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - path := index.Path - if err := json.Unmarshal(jsonData, index); err != nil { - return err - } - index.Path = path - return nil -} - -func (index *Index) save() error { - jsonData, err := json.Marshal(index) - if err != nil { - return err - } - if err := ioutil.WriteFile(index.Path, jsonData, 0600); err != nil { - return err - } - return nil -} - -// History wraps an array of images so they can be sorted by date (most recent first) - -type History []*Image - -func (history *History) Len() int { - return len(*history) -} - -func (history *History) Less(i, j int) bool { - images := *history - return images[j].Created.Before(images[i].Created) -} - -func (history *History) Swap(i, j int) { - images := *history - tmp := images[i] - images[i] = images[j] - images[j] = tmp -} - -func (history *History) Add(image *Image) { - *history = append(*history, image) - sort.Sort(history) -} - -func (history *History) Del(id string) { - for idx, image := range *history { - if image.Id == id { - *history = append((*history)[:idx], (*history)[idx+1:]...) - } - } -} - -type Image struct { - Id string // Globally unique identifier - Layers []string // Absolute paths - Created time.Time - Parent string -} - -func (image *Image) IdParts() (string, string) { - if len(image.Id) < 8 { - return "", image.Id - } - hash := image.Id[len(image.Id)-8 : len(image.Id)] - name := image.Id[:len(image.Id)-9] - return name, hash -} - -func (image *Image) IdIsFinal() bool { - return len(image.Layers) == 1 -} - -func generateImageId(name string, layers []string) (string, error) { - if len(layers) == 0 { - return "", errors.New("No layers provided.") - } - var hash string - if len(layers) == 1 { - hash = path.Base(layers[0]) - } else { - var ids string - for _, layer := range layers { - ids += path.Base(layer) - } - if h, err := future.ComputeId(strings.NewReader(ids)); err != nil { - return "", err - } else { - hash = h - } - } - return name + ":" + hash, nil -} - -func NewImage(name string, layers []string, parent string) (*Image, error) { - id, err := generateImageId(name, layers) - if err != nil { - return nil, err - } - return &Image{ - Id: id, - Layers: layers, - Created: time.Now(), - Parent: parent, - }, nil -} From 687d6f25eeb8f4bc505dc9c6c0d9ea305c824e2d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 11 Mar 2013 02:59:52 -0700 Subject: [PATCH 17/50] Change relative paths to absolute --- client/client.go | 4 ++-- container.go | 2 +- docker.go | 2 +- docker/docker.go | 2 +- dockerd/dockerd.go | 4 ++-- fs/layers.go | 2 +- fs/store_test.go | 2 +- server/server.go | 8 ++++---- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/client.go b/client/client.go index a277a4b18..4c4ea1c5e 100644 --- a/client/client.go +++ b/client/client.go @@ -1,8 +1,8 @@ package client import ( - "../future" - "../rcli" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "io" "io/ioutil" "log" diff --git a/container.go b/container.go index 52a330621..b99bf105e 100644 --- a/container.go +++ b/container.go @@ -1,7 +1,7 @@ package docker import ( - "./fs" + "github.com/dotcloud/docker/fs" "encoding/json" "errors" "github.com/kr/pty" diff --git a/docker.go b/docker.go index 49c6d8324..2561d73e1 100644 --- a/docker.go +++ b/docker.go @@ -1,7 +1,7 @@ package docker import ( - "./fs" + "github.com/dotcloud/docker/fs" "container/list" "fmt" "io/ioutil" diff --git a/docker/docker.go b/docker/docker.go index df8a09443..73939abb6 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -1,7 +1,7 @@ package main import ( - "../client" + "github.com/dotcloud/docker/client" "flag" "log" "os" diff --git a/dockerd/dockerd.go b/dockerd/dockerd.go index 22ae8df8e..456dfe9d6 100644 --- a/dockerd/dockerd.go +++ b/dockerd/dockerd.go @@ -1,8 +1,8 @@ package main import ( - ".." - "../server" + "github.com/dotcloud/docker" + "github.com/dotcloud/docker/server" "flag" "log" ) diff --git a/fs/layers.go b/fs/layers.go index 11452fcf2..b7d97bd7f 100644 --- a/fs/layers.go +++ b/fs/layers.go @@ -1,7 +1,7 @@ package fs import ( - "../future" + "github.com/dotcloud/docker/future" "errors" "fmt" "io" diff --git a/fs/store_test.go b/fs/store_test.go index d5715d1ba..33bb16c92 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -1,7 +1,7 @@ package fs import ( - "../fake" + "github.com/dotcloud/docker/fake" "errors" "fmt" "io/ioutil" diff --git a/server/server.go b/server/server.go index c403ed187..65d4fd962 100644 --- a/server/server.go +++ b/server/server.go @@ -1,10 +1,10 @@ package server import ( - ".." - "../fs" - "../future" - "../rcli" + "github.com/dotcloud/docker" + "github.com/dotcloud/docker/fs" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "bufio" "bytes" "encoding/json" From 5d6199b9251261216cb096cd919e3cc5eb3efdd6 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 11 Mar 2013 07:39:06 -0700 Subject: [PATCH 18/50] post-merge repairs --- client/client.go | 4 ++-- container.go | 1 - docker/docker.go | 2 +- server/server.go | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/client/client.go b/client/client.go index 4c4ea1c5e..a277a4b18 100644 --- a/client/client.go +++ b/client/client.go @@ -1,8 +1,8 @@ package client import ( - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/rcli" + "../future" + "../rcli" "io" "io/ioutil" "log" diff --git a/container.go b/container.go index 21f4d5492..52a330621 100644 --- a/container.go +++ b/container.go @@ -2,7 +2,6 @@ package docker import ( "./fs" - "bytes" "encoding/json" "errors" "github.com/kr/pty" diff --git a/docker/docker.go b/docker/docker.go index fa9011def..df8a09443 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -1,8 +1,8 @@ package main import ( + "../client" "flag" - "github.com/dotcloud/docker/client" "log" "os" "path" diff --git a/server/server.go b/server/server.go index b091913e0..c403ed187 100644 --- a/server/server.go +++ b/server/server.go @@ -434,7 +434,7 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } } fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.images.Create(resp.Body, nil, name, "") + img, err := srv.images.Create(archive, nil, name, "") if err != nil { return err } @@ -614,7 +614,7 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } if container := srv.containers.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := docker.Tar(container.Mountpoint.Rw) + rwTar, err := fs.Tar(container.Mountpoint.Rw, fs.Uncompressed) if err != nil { return err } @@ -651,7 +651,7 @@ func (srv *Server) CmdTar(stdin io.ReadCloser, stdout io.Writer, args ...string) if err := container.Mountpoint.EnsureMounted(); err != nil { return err } - data, err := docker.Tar(container.Mountpoint.Root) + data, err := fs.Tar(container.Mountpoint.Root, fs.Uncompressed) if err != nil { return err } From a3a946703ba93e7d937680897ed89eb99c52acef Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 14:28:11 -0700 Subject: [PATCH 19/50] Set the memory soft limit to the same value than the hard limit --- lxc_template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lxc_template.go b/lxc_template.go index 931095c99..47f2058dc 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -87,6 +87,7 @@ lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw se # limits {{if .Config.Ram}} lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} +lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Ram}} {{end}} ` From 06d420d2ead65cb5af760d7946d8844f568f4d65 Mon Sep 17 00:00:00 2001 From: Charles Hooper Date: Mon, 11 Mar 2013 22:08:22 +0000 Subject: [PATCH 20/50] Fix CmdInspect and CmdImages help/parsing. Fixes #38 --- server/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index bc642cdbb..9efdaa97d 100644 --- a/server/server.go +++ b/server/server.go @@ -60,6 +60,7 @@ func (srv *Server) Help() string { {"tar", "Stream the contents of a container as a tar archive"}, {"web", "Generate a web UI"}, {"images", "List images"}, + {"inspect", "Return low-level information on a container"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd...) } @@ -282,7 +283,6 @@ func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -463,7 +463,9 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") - cmd.Parse(args) + if err := cmd.Parse(args); err != nil { + return nil + } if cmd.NArg() > 1 { cmd.Usage() return nil @@ -893,7 +895,7 @@ func New() (*Server, error) { return nil, err } srv := &Server{ - images: images, + images: images, containers: containers, } return srv, nil From a7cb3cb86a39f950a517b61e3a674e0cd080804f Mon Sep 17 00:00:00 2001 From: Charles Hooper Date: Mon, 11 Mar 2013 22:29:13 +0000 Subject: [PATCH 21/50] Complete fix for #38 -- fix double-usage on help --- server/server.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/server/server.go b/server/server.go index 9efdaa97d..3a66d7e03 100644 --- a/server/server.go +++ b/server/server.go @@ -71,7 +71,6 @@ func (srv *Server) Help() string { func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] NAME", "Block until a container stops, then print its exit code.") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -100,7 +99,6 @@ func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "stop", "[OPTIONS] NAME", "Stop a running container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -123,7 +121,6 @@ func (srv *Server) CmdStop(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "restart", "[OPTIONS] NAME", "Restart a running container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -146,7 +143,6 @@ func (srv *Server) CmdRestart(stdin io.ReadCloser, stdout io.Writer, args ...str func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "start", "[OPTIONS] NAME", "Start a stopped container") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -169,7 +165,6 @@ func (srv *Server) CmdStart(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "umount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -192,7 +187,6 @@ func (srv *Server) CmdUmount(stdin io.ReadCloser, stdout io.Writer, args ...stri func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "umount", "[OPTIONS] NAME", "mount a container's filesystem (debug only)") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { @@ -215,7 +209,6 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -237,7 +230,6 @@ func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -259,7 +251,6 @@ func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...strin func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 2 { @@ -315,7 +306,6 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "port", "[OPTIONS] CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() != 2 { @@ -341,7 +331,6 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") if err := cmd.Parse(args); err != nil { - cmd.Usage() return nil } if cmd.NArg() < 1 { From 52b811f50a0906904d688e5fae8f32e1ada90f1a Mon Sep 17 00:00:00 2001 From: Charles Hooper Date: Mon, 11 Mar 2013 23:11:46 +0000 Subject: [PATCH 22/50] Add all commands to the output of 'dotcloud help'. Fixes #39 --- server/server.go | 54 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/server/server.go b/server/server.go index 3a66d7e03..37404f2c2 100644 --- a/server/server.go +++ b/server/server.go @@ -37,30 +37,40 @@ func (srv *Server) Name() string { return "docker" } +// FIXME: Stop violating DRY by repeating usage here and in Subcmd declarations func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]interface{}{ - {"run", "Run a command in a container"}, - {"ps", "Display a list of containers"}, - {"pull", "Download a tarball and create a container from it"}, - {"put", "Upload a tarball and create a container from it"}, - {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, - {"rm", "Remove containers"}, - {"kill", "Kill a running container"}, - {"wait", "Wait for the state of a container to change"}, - {"stop", "Stop a running container"}, - {"start", "Start a stopped container"}, - {"restart", "Restart a running container"}, - {"logs", "Fetch the logs of a container"}, + {"attach", "Attach to a running container"}, + {"cat", "Write the contents of a container's file to standard output"}, + {"commit", "Create a new image from a container's changes"}, + {"cp", "Create a copy of IMAGE and call it NAME"}, + {"debug", "(debug only) (No documentation available)"}, {"diff", "Inspect changes on a container's filesystem"}, - {"commit", "Save the state of a container"}, - {"attach", "Attach to the standard inputs and outputs of a running container"}, - {"wait", "Block until a container exits, then print its exit code"}, - {"info", "Display system-wide information"}, - {"tar", "Stream the contents of a container as a tar archive"}, - {"web", "Generate a web UI"}, {"images", "List images"}, + {"info", "Display system-wide information"}, {"inspect", "Return low-level information on a container"}, + {"kill", "Kill a running container"}, + {"layers", "(debug only) List filesystem layers"}, + {"logs", "Fetch the logs of a container"}, + {"ls", "List the contents of a container's directory"}, + {"mirror", "(debug only) (No documentation available)"}, + {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, + {"ps", "List containers"}, + {"pull", "Download a new image from a remote location"}, + {"put", "Import a new image from a local archive"}, + {"reset", "Reset changes to a container's filesystem"}, + {"restart", "Restart a running container"}, + {"rm", "Remove a container"}, + {"rmimage", "Remove an image"}, + {"run", "Run a command in a new container"}, + {"start", "Start a stopped container"}, + {"stop", "Stop a running container"}, + {"tar", "Stream the contents of a container as a tar archive"}, + {"umount", "(debug only) Mount a container's filesystem"}, + {"wait", "Block until a container stops, then print its exit code"}, + {"web", "A web UI for docker"}, + {"write", "Write the contents of standard input to a container's file"}, } { help += fmt.Sprintf(" %-10.10s%s\n", cmd...) } @@ -89,6 +99,14 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "info", "", "Display system-wide information.") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() > 1 { + cmd.Usage() + return nil + } fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, From 2548722dae1be15c61ee8db1a5eaa1d04ba7fcef Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 11 Mar 2013 17:05:01 -0700 Subject: [PATCH 23/50] Fix a bug which caused dockerd to crash at startup if a container didn't have a registered mountpoint --- container.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index b99bf105e..aa1acc3c4 100644 --- a/container.go +++ b/container.go @@ -1,9 +1,9 @@ package docker import ( - "github.com/dotcloud/docker/fs" "encoding/json" "errors" + "github.com/dotcloud/docker/fs" "github.com/kr/pty" "io" "io/ioutil" @@ -129,6 +129,8 @@ func loadContainer(store *fs.Store, containerPath string, netManager *NetworkMan ) if err != nil { return nil, err + } else if mountpoint == nil { + return nil, errors.New("Couldn't load container: unregistered mountpoint.") } container := &Container{ stdout: newWriteBroadcaster(), From 75d04a5a7561cabba0aaf2875ef4479381096f91 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 17:40:54 -0700 Subject: [PATCH 24/50] Added support for RamSwap in the generated LXC config (to limit the swap and have the right default settings) --- container.go | 3 ++- lxc_template.go | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/container.go b/container.go index 530c08bc3..c83750054 100644 --- a/container.go +++ b/container.go @@ -53,7 +53,8 @@ type Container struct { type Config struct { Hostname string User string - Ram int64 + Ram int64 // Memory limit (in bytes) + RamSwap int64 // Total memory usage (ram + swap); set `-1' to disable swap Ports []int Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin diff --git a/lxc_template.go b/lxc_template.go index 47f2058dc..5a37624a3 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -88,14 +88,29 @@ lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw se {{if .Config.Ram}} lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Ram}} +{{with $ramSwap := getRamSwap .Config}} +lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} +{{end}} {{end}} ` var LxcTemplateCompiled *template.Template +func getRamSwap(config *Config) int64 { + // By default, RamSwap is set to twice the size of RAM. + // If you want to omit RamSwap, set it to `-1'. + if config.RamSwap < 0 { + return 0 + } + return config.Ram * 2 +} + func init() { var err error - LxcTemplateCompiled, err = template.New("lxc").Parse(LxcTemplate) + funcMap := template.FuncMap{ + "getRamSwap": getRamSwap, + } + LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) if err != nil { panic(err) } From 43875a73a0c2ee51f8bcd36634bbf64d836df6f8 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Mon, 11 Mar 2013 17:47:28 -0700 Subject: [PATCH 25/50] added a fix to the upstart command that wasn't setting hte LOCALE correctly, which was causing issues with UTF-8 bundles --- puppet/modules/docker/templates/dockerd.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/puppet/modules/docker/templates/dockerd.conf b/puppet/modules/docker/templates/dockerd.conf index a00d322bc..c05f60639 100644 --- a/puppet/modules/docker/templates/dockerd.conf +++ b/puppet/modules/docker/templates/dockerd.conf @@ -7,5 +7,6 @@ start on runlevel [3] respawn script - /home/vagrant/dockerd/dockerd + test -f /etc/default/locale && . /etc/default/locale || true + LANG=$LANG LC_ALL=$LANG /usr/local/bin/dockerd end script From d4da0a0ddbb3ec1979941bc3d2425aab3a6b3f19 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Mon, 11 Mar 2013 17:48:39 -0700 Subject: [PATCH 26/50] changed the docker version in vagrant to pull down latest version, moved some things around and put docker and dockerd into /usr/local/bin now. Also added git to install package list --- puppet/modules/docker/manifests/init.pp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/puppet/modules/docker/manifests/init.pp b/puppet/modules/docker/manifests/init.pp index 2923b302f..ed3dd62b6 100644 --- a/puppet/modules/docker/manifests/init.pp +++ b/puppet/modules/docker/manifests/init.pp @@ -1,14 +1,15 @@ class docker { # update this with latest docker binary distro - $docker_url = "https://dl.dropbox.com/u/20637798/docker.tar.gz" + $docker_url = "http://docker.io.s3.amazonaws.com/builds/$kernel/$hardwaremodel/docker-master.tgz" # update this with latest go binary distry $go_url = "http://go.googlecode.com/files/go1.0.3.linux-amd64.tar.gz" - Package { ensure => "installed" } - package { ["lxc", "debootstrap", "wget", "bsdtar"]: } + package { ["lxc", "debootstrap", "wget", "bsdtar", "git"]: } + + notify { "docker_url = $docker_url": withpath => true } exec { "debootstrap" : require => Package["debootstrap"], @@ -26,7 +27,7 @@ class docker { exec { "fetch-docker" : require => Package["wget"], command => "/usr/bin/wget -O - $docker_url | /bin/tar xz -C /home/vagrant", - creates => "/home/vagrant/docker/dockerd" + creates => "/home/vagrant/docker-master" } file { "/etc/init/dockerd.conf": @@ -39,10 +40,16 @@ class docker { exec { "copy-docker-bin" : require => Exec["fetch-docker"], - command => "/bin/cp /home/vagrant/docker/docker /usr/local/bin", + command => "/bin/cp /home/vagrant/docker-master/docker /usr/local/bin", creates => "/usr/local/bin/docker" } + exec { "copy-dockerd-bin" : + require => Exec["fetch-docker"], + command => "/bin/cp /home/vagrant/docker-master/dockerd /usr/local/bin", + creates => "/usr/local/bin/dockerd" + } + service { "dockerd" : ensure => "running", start => "/sbin/initctl start dockerd", From 4e5ae883722ad814ef19e3f9d333f6218db752a5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:15:29 -0700 Subject: [PATCH 27/50] Implemented unit tests for the generated LXC config --- container_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++ rcli/http.go | 9 +++----- rcli/tcp.go | 12 +++++------ rcli/types.go | 10 ++++----- 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/container_test.go b/container_test.go index 8c187ecb9..f79abe336 100644 --- a/container_test.go +++ b/container_test.go @@ -1,9 +1,12 @@ package docker import ( + "bufio" "fmt" "io" "io/ioutil" + "math/rand" + "os" "sort" "strings" "testing" @@ -561,6 +564,58 @@ func TestEnv(t *testing.T) { } } +func grepFile(t *testing.T, path string, pattern string) { + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + r := bufio.NewReader(f) + var ( + line string + ) + err = nil + for err == nil { + line, err = r.ReadString('\n') + if strings.Contains(line, pattern) == true { + return + } + } + t.Fatalf("grepFile: pattern \"%s\" not found in \"%s\"", pattern, path) +} + +func TestLXCConfig(t *testing.T) { + docker, err := newTestDocker() + if err != nil { + t.Fatal(err) + } + // Ram is allocated randomly for testing + rand.Seed(time.Now().UTC().UnixNano()) + ramMin := 33554432 + ramMax := 536870912 + ram := ramMin + rand.Intn(ramMax-ramMin) + container, err := docker.Create( + "config_test", + "/bin/true", + []string{}, + []string{testLayerPath}, + &Config{ + Hostname: "foobar", + Ram: int64(ram), + }, + ) + if err != nil { + t.Fatal(err) + } + defer docker.Destroy(container) + container.generateLXCConfig() + grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") + grepFile(t, container.lxcConfigPath, + fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", ram)) + grepFile(t, container.lxcConfigPath, + fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", ram*2)) +} + func BenchmarkRunSequencial(b *testing.B) { docker, err := newTestDocker() if err != nil { diff --git a/rcli/http.go b/rcli/http.go index e6cb5657d..cc8d3b149 100644 --- a/rcli/http.go +++ b/rcli/http.go @@ -1,13 +1,12 @@ package rcli import ( + "fmt" "net/http" "net/url" "path" - "fmt" ) - // Use this key to encode an RPC call into an URL, // eg. domain.tld/path/to/method?q=get_user&q=gordon const ARG_URL_KEY = "q" @@ -16,18 +15,16 @@ func URLToCall(u *url.URL) (method string, args []string) { return path.Base(u.Path), u.Query()[ARG_URL_KEY] } - func ListenAndServeHTTP(addr string, service Service) error { return http.ListenAndServe(addr, http.HandlerFunc( - func (w http.ResponseWriter, r *http.Request) { + func(w http.ResponseWriter, r *http.Request) { cmd, args := URLToCall(r.URL) if err := call(service, r.Body, &AutoFlush{w}, append([]string{cmd}, args...)...); err != nil { - fmt.Fprintf(w, "Error: " + err.Error() + "\n") + fmt.Fprintf(w, "Error: "+err.Error()+"\n") } })) } - type AutoFlush struct { http.ResponseWriter } diff --git a/rcli/tcp.go b/rcli/tcp.go index 0a06d459c..869a3bcdb 100644 --- a/rcli/tcp.go +++ b/rcli/tcp.go @@ -1,13 +1,13 @@ package rcli import ( + "bufio" + "encoding/json" + "fmt" "io" "io/ioutil" - "net" "log" - "fmt" - "encoding/json" - "bufio" + "net" ) // Connect to a remote endpoint using protocol `proto` and address `addr`, @@ -44,7 +44,7 @@ func ListenAndServe(proto, addr string, service Service) error { go func() { if err := Serve(conn, service); err != nil { log.Printf("Error: " + err.Error() + "\n") - fmt.Fprintf(conn, "Error: " + err.Error() + "\n") + fmt.Fprintf(conn, "Error: "+err.Error()+"\n") } conn.Close() }() @@ -53,7 +53,6 @@ func ListenAndServe(proto, addr string, service Service) error { return nil } - // Parse an rcli call on a new connection, and pass it to `service` if it // is valid. func Serve(conn io.ReadWriter, service Service) error { @@ -68,4 +67,3 @@ func Serve(conn io.ReadWriter, service Service) error { } return nil } - diff --git a/rcli/types.go b/rcli/types.go index b8572cd89..52079291b 100644 --- a/rcli/types.go +++ b/rcli/types.go @@ -8,13 +8,13 @@ package rcli // are the usual suspects. import ( + "errors" + "flag" "fmt" "io" - "reflect" - "flag" "log" + "reflect" "strings" - "errors" ) type Service interface { @@ -25,7 +25,6 @@ type Service interface { type Cmd func(io.ReadCloser, io.Writer, ...string) error type CmdMethod func(Service, io.ReadCloser, io.Writer, ...string) error - func call(service Service, stdin io.ReadCloser, stdout io.Writer, args ...string) error { if len(args) == 0 { args = []string{"help"} @@ -63,7 +62,7 @@ func getMethod(service Service, name string) Cmd { return nil } } - methodName := "Cmd"+strings.ToUpper(name[:1])+strings.ToLower(name[1:]) + methodName := "Cmd" + strings.ToUpper(name[:1]) + strings.ToLower(name[1:]) method, exists := reflect.TypeOf(service).MethodByName(methodName) if !exists { return nil @@ -91,4 +90,3 @@ func Subcmd(output io.Writer, name, signature, description string) *flag.FlagSet } return flags } - From 948961831ab6cfb0c94e1e80552f4438d1f434a7 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:25:02 -0700 Subject: [PATCH 28/50] Renamed Container property Ram to Memory before it is too late --- container.go | 14 +++++++------- container_test.go | 18 +++++++++--------- lxc_template.go | 20 ++++++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/container.go b/container.go index c83750054..9aad176ed 100644 --- a/container.go +++ b/container.go @@ -51,13 +51,13 @@ type Container struct { } type Config struct { - Hostname string - User string - Ram int64 // Memory limit (in bytes) - RamSwap int64 // Total memory usage (ram + swap); set `-1' to disable swap - Ports []int - Tty bool // Attach standard streams to a tty, including stdin if it is not closed. - OpenStdin bool // Open stdin + Hostname string + User string + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + Ports []int + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin } type NetworkSettings struct { diff --git a/container_test.go b/container_test.go index f79abe336..f9c835edf 100644 --- a/container_test.go +++ b/container_test.go @@ -24,7 +24,7 @@ func TestStart(t *testing.T) { []string{"-al"}, []string{testLayerPath}, &Config{ - Ram: 33554432, + Memory: 33554432, }, ) if err != nil { @@ -60,7 +60,7 @@ func TestRun(t *testing.T) { []string{"-al"}, []string{testLayerPath}, &Config{ - Ram: 33554432, + Memory: 33554432, }, ) if err != nil { @@ -589,11 +589,11 @@ func TestLXCConfig(t *testing.T) { if err != nil { t.Fatal(err) } - // Ram is allocated randomly for testing + // Memory is allocated randomly for testing rand.Seed(time.Now().UTC().UnixNano()) - ramMin := 33554432 - ramMax := 536870912 - ram := ramMin + rand.Intn(ramMax-ramMin) + memMin := 33554432 + memMax := 536870912 + mem := memMin + rand.Intn(memMax-memMin) container, err := docker.Create( "config_test", "/bin/true", @@ -601,7 +601,7 @@ func TestLXCConfig(t *testing.T) { []string{testLayerPath}, &Config{ Hostname: "foobar", - Ram: int64(ram), + Memory: int64(mem), }, ) if err != nil { @@ -611,9 +611,9 @@ func TestLXCConfig(t *testing.T) { container.generateLXCConfig() grepFile(t, container.lxcConfigPath, "lxc.utsname = foobar") grepFile(t, container.lxcConfigPath, - fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", ram)) + fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) grepFile(t, container.lxcConfigPath, - fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", ram*2)) + fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2)) } func BenchmarkRunSequencial(b *testing.B) { diff --git a/lxc_template.go b/lxc_template.go index 5a37624a3..5d92c9e6d 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -85,10 +85,10 @@ lxc.mount.entry = /etc/resolv.conf {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config # limits -{{if .Config.Ram}} -lxc.cgroup.memory.limit_in_bytes = {{.Config.Ram}} -lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Ram}} -{{with $ramSwap := getRamSwap .Config}} +{{if .Config.Memory}} +lxc.cgroup.memory.limit_in_bytes = {{.Config.Memory}} +lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Memory}} +{{with $ramSwap := getMemorySwap .Config}} lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} {{end}} {{end}} @@ -96,19 +96,19 @@ lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} var LxcTemplateCompiled *template.Template -func getRamSwap(config *Config) int64 { - // By default, RamSwap is set to twice the size of RAM. - // If you want to omit RamSwap, set it to `-1'. - if config.RamSwap < 0 { +func getMemorySwap(config *Config) int64 { + // By default, MemorySwap is set to twice the size of RAM. + // If you want to omit MemorySwap, set it to `-1'. + if config.MemorySwap < 0 { return 0 } - return config.Ram * 2 + return config.Memory * 2 } func init() { var err error funcMap := template.FuncMap{ - "getRamSwap": getRamSwap, + "getMemorySwap": getMemorySwap, } LxcTemplateCompiled, err = template.New("lxc").Funcs(funcMap).Parse(LxcTemplate) if err != nil { From 3684b67572605de492abba041695bd80ef42bcb5 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:51:24 -0700 Subject: [PATCH 29/50] Added -m to the run command in order to set a memory limit to a container --- server/server.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index bc642cdbb..a490641d2 100644 --- a/server/server.go +++ b/server/server.go @@ -712,10 +712,18 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *image.Image, ports []int, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *image.Image, ports []int, user string, + tty bool, openStdin bool, memory int64, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] container, err := srv.containers.Create(id, cmd, args, img.Layers, - &docker.Config{Hostname: id, Ports: ports, User: user, Tty: tty, OpenStdin: openStdin}) + &docker.Config{ + Hostname: id, + Ports: ports, + User: user, + Tty: tty, + OpenStdin: openStdin, + Memory: memory, + }) if err != nil { return nil, err } @@ -799,6 +807,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") fl_comment := cmd.String("c", "", "Comment") + fl_memory := cmd.Int64("m", 0, "Memory limit (in bytes)") var fl_ports ports cmd.Var(&fl_ports, "p", "Map a network port to the container") if err := cmd.Parse(args); err != nil { @@ -826,7 +835,8 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) return errors.New("No such image: " + name) } // Create new container - container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, *fl_stdin, *fl_comment, cmdline[0], cmdline[1:]...) + container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, + *fl_stdin, *fl_memory, *fl_comment, cmdline[0], cmdline[1:]...) if err != nil { return errors.New("Error creating container: " + err.Error()) } From f8fee421819e3c272e1c20890da7c90b4da971ad Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 11 Mar 2013 19:55:14 -0700 Subject: [PATCH 30/50] Missed a rename --- lxc_template.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lxc_template.go b/lxc_template.go index 5d92c9e6d..2a60e8b96 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -88,8 +88,8 @@ lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw se {{if .Config.Memory}} lxc.cgroup.memory.limit_in_bytes = {{.Config.Memory}} lxc.cgroup.memory.soft_limit_in_bytes = {{.Config.Memory}} -{{with $ramSwap := getMemorySwap .Config}} -lxc.cgroup.memory.memsw.limit_in_bytes = {{$ramSwap}} +{{with $memSwap := getMemorySwap .Config}} +lxc.cgroup.memory.memsw.limit_in_bytes = {{$memSwap}} {{end}} {{end}} ` From 132ecb24822072c82718d47ac23da62a8ea774fc Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 11 Mar 2013 21:04:16 -0700 Subject: [PATCH 31/50] Replaced 'pull' and 'put' with 'import' --- server/server.go | 78 ++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/server/server.go b/server/server.go index 65d4fd962..00ef9c496 100644 --- a/server/server.go +++ b/server/server.go @@ -1,15 +1,15 @@ package server import ( - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/fs" - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/rcli" "bufio" "bytes" "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker" + "github.com/dotcloud/docker/fs" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "io" "net/http" "net/url" @@ -42,8 +42,7 @@ func (srv *Server) Help() string { for _, cmd := range [][]interface{}{ {"run", "Run a command in a container"}, {"ps", "Display a list of containers"}, - {"pull", "Download a tarball and create a container from it"}, - {"put", "Upload a tarball and create a container from it"}, + {"import", "Create a new filesystem image from the contents of a tarball"}, {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"rm", "Remove containers"}, {"kill", "Kill a running container"}, @@ -401,36 +400,42 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } -func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] NAME", "Download a new image from a remote location") +func (srv *Server) CmdImport(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "import", "[OPTIONS] NAME", "Create a new filesystem image from the contents of a tarball") + fl_stdin := cmd.Bool("stdin", false, "Read tarball from stdin") if err := cmd.Parse(args); err != nil { return nil } + var archive io.Reader name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") } - u, err := url.Parse(name) - if err != nil { - return err - } - if u.Scheme == "" { - u.Scheme = "http" - } - // FIXME: hardcode a mirror URL that does not depend on a single provider. - if u.Host == "" { - u.Host = "s3.amazonaws.com" - u.Path = path.Join("/docker.io/images", u.Path) - } - fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) - // Download with curl (pretty progress bar) - // If curl is not available, fallback to http.Get() - archive, err := future.Curl(u.String(), stdout) - if err != nil { - if resp, err := http.Get(u.String()); err != nil { + if *fl_stdin { + archive = stdin + } else { + u, err := url.Parse(name) + if err != nil { return err - } else { - archive = resp.Body + } + if u.Scheme == "" { + u.Scheme = "http" + } + // FIXME: hardcode a mirror URL that does not depend on a single provider. + if u.Host == "" { + u.Host = "s3.amazonaws.com" + u.Path = path.Join("/docker.io/images", u.Path) + } + fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) + // Download with curl (pretty progress bar) + // If curl is not available, fallback to http.Get() + archive, err = future.Curl(u.String(), stdout) + if err != nil { + if resp, err := http.Get(u.String()); err != nil { + return err + } else { + archive = resp.Body + } } } fmt.Fprintf(stdout, "Unpacking to %s\n", name) @@ -442,23 +447,6 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } -func (srv *Server) CmdPut(stdin io.ReadCloser, stdout io.Writer, args ...string) error { - cmd := rcli.Subcmd(stdout, "put", "[OPTIONS] NAME", "Import a new image from a local archive.") - if err := cmd.Parse(args); err != nil { - return nil - } - name := cmd.Arg(0) - if name == "" { - return errors.New("Not enough arguments") - } - img, err := srv.images.Create(stdin, nil, name, "") - if err != nil { - return err - } - fmt.Fprintln(stdout, img.Id) - return nil -} - func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "images", "[OPTIONS] [NAME]", "List images") limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") From 53e2e8209fd882a3b4f073ce30dec444cc9c7cc0 Mon Sep 17 00:00:00 2001 From: jpetazzo Date: Mon, 11 Mar 2013 19:03:22 -0700 Subject: [PATCH 32/50] Add missing shebang cloud-init expects a shebang in the beginning of the script --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index af5c214d7..52cd3c0a0 100644 --- a/install.sh +++ b/install.sh @@ -1,4 +1,5 @@ -# This script is meant for quick & easy install via 'curl URL-OF-SCRIPPT | bash' +#!/bin/sh +# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' # Courtesy of Jeff Lindsay cd /tmp From af2aac8736d71fe79492f9abf3a16fba97857eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:07:16 -0700 Subject: [PATCH 33/50] Stream and unpack the tarball on the fly We don't need temporary files. Also, get the tarball from get.docker.io instead of Dropbox. --- install.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/install.sh b/install.sh index 52cd3c0a0..2ecb8b47a 100644 --- a/install.sh +++ b/install.sh @@ -2,18 +2,13 @@ # This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' # Courtesy of Jeff Lindsay -cd /tmp - echo "Ensuring dependencies are installed..." apt-get --yes install lxc wget bsdtar 2>&1 > /dev/null -echo "Downloading docker binary..." -wget -q https://dl.dropbox.com/u/20637798/docker.tar.gz 2>&1 > /dev/null -tar -xf docker.tar.gz 2>&1 > /dev/null - -echo "Installing into /usr/local/bin..." -mv docker/docker /usr/local/bin -mv dockerd/dockerd /usr/local/bin +echo "Downloading docker binary and uncompressing into /usr/local/bin..." +curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | +tar -C /usr/local/bin --strip-components=1 -zxf- \ +docker-master/docker docker-master/dockerd if [[ -f /etc/init/dockerd.conf ]] then @@ -26,10 +21,5 @@ fi echo "Restarting dockerd..." restart dockerd > /dev/null -echo "Cleaning up..." -rmdir docker -rmdir dockerd -rm docker.tar.gz - echo "Finished!" echo From fda456b82e51224fe4e7ae6c0bc2b0f0812729ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:08:21 -0700 Subject: [PATCH 34/50] Use "start" instead of "restart" Because when the job isn't running, "restart" won't work. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 2ecb8b47a..43e8ecce1 100644 --- a/install.sh +++ b/install.sh @@ -18,8 +18,8 @@ else echo "exec /usr/local/bin/dockerd" > /etc/init/dockerd.conf fi -echo "Restarting dockerd..." -restart dockerd > /dev/null +echo "Starting dockerd..." +start dockerd > /dev/null echo "Finished!" echo From c1fa4efad66a1ae8cbccd7c55bc195be03d94b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:09:19 -0700 Subject: [PATCH 35/50] Try to load, or install, AUFS modules When running precise or quantal, installing the matching -extra kernel will install the required file. --- install.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 43e8ecce1..4e592e5dc 100644 --- a/install.sh +++ b/install.sh @@ -2,8 +2,34 @@ # This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' # Courtesy of Jeff Lindsay -echo "Ensuring dependencies are installed..." -apt-get --yes install lxc wget bsdtar 2>&1 > /dev/null +echo "Ensuring basic dependencies are installed..." +apt-get -qq update +apt-get -qq install lxc wget bsdtar + +echo "Looking in /proc/filesystems to see if we have AUFS support..." +if grep -q aufs /proc/filesystems +then + echo "Found." +else + echo "Ahem, it looks like the current kernel does not support AUFS." + echo "Let's see if we can load the AUFS module with modprobe..." + if modprobe aufs + then + echo "Module loaded." + else + echo "Ahem, things didn't turn out as expected." + KPKG=linux-image-extra-$(uname -r) + echo "Trying to install $KPKG..." + if apt-get -qq install $KPKG + then + echo "Installed." + else + echo "Oops, we couldn't install the -extra kernel." + echo "Are you sure you are running a supported version of Ubuntu?" + echo "Proceeding anyway, but Docker will probably NOT WORK!" + fi + fi +fi echo "Downloading docker binary and uncompressing into /usr/local/bin..." curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | From f90f6a0b9248489c43188ca79b9e0dfc306fd8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:13:08 -0700 Subject: [PATCH 36/50] Get rid of bashism --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 4e592e5dc..f8dca1747 100644 --- a/install.sh +++ b/install.sh @@ -1,5 +1,5 @@ #!/bin/sh -# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | bash' +# This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | sh' # Courtesy of Jeff Lindsay echo "Ensuring basic dependencies are installed..." @@ -36,7 +36,7 @@ curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | tar -C /usr/local/bin --strip-components=1 -zxf- \ docker-master/docker docker-master/dockerd -if [[ -f /etc/init/dockerd.conf ]] +if [ -f /etc/init/dockerd.conf ] then echo "Upstart script already exists." else From b728c1a81fbb441442d635788e814d47436c613f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 11 Mar 2013 23:37:38 -0700 Subject: [PATCH 37/50] Add script upload instructions --- install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index f8dca1747..0cfba4ddc 100644 --- a/install.sh +++ b/install.sh @@ -1,6 +1,10 @@ #!/bin/sh # This script is meant for quick & easy install via 'curl URL-OF-SCRIPT | sh' -# Courtesy of Jeff Lindsay +# Original version by Jeff Lindsay +# Revamped by Jerome Petazzoni +# +# This script canonical location is http://get.docker.io/; to update it, run: +# s3cmd put -m text/x-shellscript -P install.sh s3://get.docker.io/index echo "Ensuring basic dependencies are installed..." apt-get -qq update @@ -47,5 +51,5 @@ fi echo "Starting dockerd..." start dockerd > /dev/null -echo "Finished!" +echo "Done." echo From e0b09d4933a289da01508039995bd627ff1392df Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 12 Mar 2013 00:08:41 -0700 Subject: [PATCH 38/50] Add "nuke" function to docker tests to avoid disk space waste --- container.go | 2 +- container_test.go | 14 ++++++++++++++ docker.go | 2 +- docker_test.go | 9 +++++++++ fs/layers.go | 2 +- fs/store.go | 2 +- state.go | 2 +- 7 files changed, 28 insertions(+), 5 deletions(-) diff --git a/container.go b/container.go index aa1acc3c4..62d7ce064 100644 --- a/container.go +++ b/container.go @@ -3,7 +3,7 @@ package docker import ( "encoding/json" "errors" - "github.com/dotcloud/docker/fs" + "./fs" "github.com/kr/pty" "io" "io/ioutil" diff --git a/container_test.go b/container_test.go index 0ffa65527..a2c49006e 100644 --- a/container_test.go +++ b/container_test.go @@ -15,6 +15,7 @@ func TestStart(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "start_test", "ls", @@ -51,6 +52,7 @@ func TestRun(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "run_test", "ls", @@ -81,6 +83,7 @@ func TestOutput(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "output_test", "echo", @@ -106,6 +109,7 @@ func TestKill(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "stop_test", "cat", @@ -148,6 +152,7 @@ func TestExitCode(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) trueContainer, err := docker.Create( "exit_test_1", @@ -193,6 +198,7 @@ func TestRestart(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "restart_test", "echo", @@ -227,6 +233,7 @@ func TestRestartStdin(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "restart_stdin_test", "cat", @@ -276,6 +283,7 @@ func TestUser(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) // Default user must be root container, err := docker.Create( @@ -393,6 +401,7 @@ func TestMultipleContainers(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container1, err := docker.Create( "container1", @@ -449,6 +458,7 @@ func TestStdin(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "stdin_test", "cat", @@ -484,6 +494,7 @@ func TestTty(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "tty_test", "cat", @@ -519,6 +530,7 @@ func TestEnv(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "env_test", "/usr/bin/env", @@ -568,6 +580,7 @@ func BenchmarkRunSequencial(b *testing.B) { if err != nil { b.Fatal(err) } + defer nuke(docker) for i := 0; i < b.N; i++ { container, err := docker.Create( fmt.Sprintf("bench_%v", i), @@ -598,6 +611,7 @@ func BenchmarkRunParallel(b *testing.B) { if err != nil { b.Fatal(err) } + defer nuke(docker) var tasks []chan error diff --git a/docker.go b/docker.go index 2561d73e1..49c6d8324 100644 --- a/docker.go +++ b/docker.go @@ -1,7 +1,7 @@ package docker import ( - "github.com/dotcloud/docker/fs" + "./fs" "container/list" "fmt" "io/ioutil" diff --git a/docker_test.go b/docker_test.go index befc0a9a3..5e75906c5 100644 --- a/docker_test.go +++ b/docker_test.go @@ -11,6 +11,10 @@ import ( const testLayerPath string = "/var/lib/docker/docker-ut.tar" +func nuke(docker *Docker) error { + return os.RemoveAll(docker.root) +} + func layerArchive(tarfile string) (io.Reader, error) { // FIXME: need to close f somewhere f, err := os.Open(tarfile) @@ -72,6 +76,7 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) // Make sure we start we 0 containers if len(docker.List()) != 0 { @@ -125,6 +130,7 @@ func TestDestroy(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container, err := docker.Create( "test_destroy", "ls", @@ -173,6 +179,7 @@ func TestGet(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker) container1, err := docker.Create( "test1", "ls", @@ -233,6 +240,7 @@ func TestRestore(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker1) if layer, err := layerArchive(testLayerPath); err != nil { panic(err) @@ -268,6 +276,7 @@ func TestRestore(t *testing.T) { if err != nil { t.Fatal(err) } + defer nuke(docker2) if len(docker2.List()) != 1 { t.Errorf("Expected 1 container, %v found", len(docker2.List())) } diff --git a/fs/layers.go b/fs/layers.go index b7d97bd7f..11452fcf2 100644 --- a/fs/layers.go +++ b/fs/layers.go @@ -1,7 +1,7 @@ package fs import ( - "github.com/dotcloud/docker/future" + "../future" "errors" "fmt" "io" diff --git a/fs/store.go b/fs/store.go index c6612d9b1..ef275ecdd 100644 --- a/fs/store.go +++ b/fs/store.go @@ -4,7 +4,7 @@ import ( "database/sql" "errors" "fmt" - "github.com/dotcloud/docker/future" + "../future" _ "github.com/mattn/go-sqlite3" "github.com/shykes/gorp" //Forked to implement CreateTablesOpts "io" diff --git a/state.go b/state.go index e864f304f..e88298ec4 100644 --- a/state.go +++ b/state.go @@ -2,7 +2,7 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/future" + "./future" "sync" "time" ) From 63dd8f101816c1253ac8d05e10f4f8a4c80f296a Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 12 Mar 2013 04:24:26 -0700 Subject: [PATCH 39/50] Re-enabled cat, write and ls ; Added creation date to image metadata --- fs/changes.go | 17 ------ fs/store.go | 20 +++++++ server/server.go | 142 +++++++++++++++++++++++++---------------------- 3 files changed, 95 insertions(+), 84 deletions(-) diff --git a/fs/changes.go b/fs/changes.go index 9bd315ad8..659f688c4 100644 --- a/fs/changes.go +++ b/fs/changes.go @@ -125,20 +125,3 @@ func (mp *Mountpoint) Reset() error { } return nil } - -// Open opens the named file for reading. -// func (fs *Filesystem) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { -// if err := fs.EnsureMounted(); err != nil { -// return nil, err -// } -// return os.OpenFile(filepath.Join(fs.RootFS, path), flag, perm) -// } - -// ReadDir reads the directory named by dirname, relative to the Filesystem's root, -// and returns a list of sorted directory entries -// func (fs *Filesystem) ReadDir(dirname string) ([]os.FileInfo, error) { -// if err := fs.EnsureMounted(); err != nil { -// return nil, err -// } -// return ioutil.ReadDir(filepath.Join(fs.RootFS, dirname)) -// } diff --git a/fs/store.go b/fs/store.go index c6612d9b1..b0fdadcc1 100644 --- a/fs/store.go +++ b/fs/store.go @@ -8,6 +8,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/shykes/gorp" //Forked to implement CreateTablesOpts "io" + "io/ioutil" "os" "path" "path/filepath" @@ -115,6 +116,7 @@ func (store *Store) Create(layerData Archive, parent *Image, pth, comment string img := &Image{ Id: future.RandomId(), Comment: comment, + Created: time.Now().Unix(), store: store, } // FIXME: we shouldn't have to pass os.Stderr to AddLayer()... @@ -163,6 +165,7 @@ type Image struct { Id string Parent string Comment string + Created int64 store *Store `db:"-"` } @@ -356,6 +359,23 @@ func (store *Store) FetchMountpoint(root, rw string) (*Mountpoint, error) { return mp, nil } +// OpenFile opens the named file for reading. +func (mp *Mountpoint) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { + if err := mp.EnsureMounted(); err != nil { + return nil, err + } + return os.OpenFile(filepath.Join(mp.Root, path), flag, perm) +} + +// ReadDir reads the directory named by dirname, relative to the Mountpoint's root, +// and returns a list of sorted directory entries +func (mp *Mountpoint) ReadDir(dirname string) ([]os.FileInfo, error) { + if err := mp.EnsureMounted(); err != nil { + return nil, err + } + return ioutil.ReadDir(filepath.Join(mp.Root, dirname)) +} + func (store *Store) AddTag(imageId, tagName string) error { if image, err := store.Get(imageId); err != nil { return err diff --git a/server/server.go b/server/server.go index c403ed187..654ca03c1 100644 --- a/server/server.go +++ b/server/server.go @@ -89,10 +89,18 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + images, _ := srv.images.Images() + var imgcount int + if images == nil { + imgcount = 0 + } else { + imgcount = len(images) + } + fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, - 0) // FIXME: Number of images + imgcount) return nil } @@ -211,73 +219,73 @@ func (srv *Server) CmdMount(stdin io.ReadCloser, stdout io.Writer, args ...strin return nil } -// func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { -// cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") -// if err := cmd.Parse(args); err != nil { -// cmd.Usage() -// return nil -// } -// if cmd.NArg() < 2 { -// cmd.Usage() -// return nil -// } -// name, path := cmd.Arg(0), cmd.Arg(1) -// if container := srv.containers.Get(name); container != nil { -// if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { -// return err -// } else if _, err := io.Copy(stdout, f); err != nil { -// return err -// } -// return nil -// } -// return errors.New("No such container: " + name) -// } +func (srv *Server) CmdCat(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "cat", "[OPTIONS] CONTAINER PATH", "write the contents of a container's file to standard output") + if err := cmd.Parse(args); err != nil { + cmd.Usage() + return nil + } + if cmd.NArg() < 2 { + cmd.Usage() + return nil + } + name, path := cmd.Arg(0), cmd.Arg(1) + if container := srv.containers.Get(name); container != nil { + if f, err := container.Mountpoint.OpenFile(path, os.O_RDONLY, 0); err != nil { + return err + } else if _, err := io.Copy(stdout, f); err != nil { + return err + } + return nil + } + return errors.New("No such container: " + name) +} -// func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { -// cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") -// if err := cmd.Parse(args); err != nil { -// cmd.Usage() -// return nil -// } -// if cmd.NArg() < 2 { -// cmd.Usage() -// return nil -// } -// name, path := cmd.Arg(0), cmd.Arg(1) -// if container := srv.containers.Get(name); container != nil { -// if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { -// return err -// } else if _, err := io.Copy(f, stdin); err != nil { -// return err -// } -// return nil -// } -// return errors.New("No such container: " + name) -// } +func (srv *Server) CmdWrite(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "write", "[OPTIONS] CONTAINER PATH", "write the contents of standard input to a container's file") + if err := cmd.Parse(args); err != nil { + cmd.Usage() + return nil + } + if cmd.NArg() < 2 { + cmd.Usage() + return nil + } + name, path := cmd.Arg(0), cmd.Arg(1) + if container := srv.containers.Get(name); container != nil { + if f, err := container.Mountpoint.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0600); err != nil { + return err + } else if _, err := io.Copy(f, stdin); err != nil { + return err + } + return nil + } + return errors.New("No such container: " + name) +} -// func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { -// cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") -// if err := cmd.Parse(args); err != nil { -// cmd.Usage() -// return nil -// } -// if cmd.NArg() < 2 { -// cmd.Usage() -// return nil -// } -// name, path := cmd.Arg(0), cmd.Arg(1) -// if container := srv.containers.Get(name); container != nil { -// if files, err := container.Mountpoint.ReadDir(path); err != nil { -// return err -// } else { -// for _, f := range files { -// fmt.Fprintln(stdout, f.Name()) -// } -// } -// return nil -// } -// return errors.New("No such container: " + name) -// } +func (srv *Server) CmdLs(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "ls", "[OPTIONS] CONTAINER PATH", "List the contents of a container's directory") + if err := cmd.Parse(args); err != nil { + cmd.Usage() + return nil + } + if cmd.NArg() < 2 { + cmd.Usage() + return nil + } + name, path := cmd.Arg(0), cmd.Arg(1) + if container := srv.containers.Get(name); container != nil { + if files, err := container.Mountpoint.ReadDir(path); err != nil { + return err + } else { + for _, f := range files { + fmt.Fprintln(stdout, f.Name()) + } + } + return nil + } + return errors.New("No such container: " + name) +} func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "inspect", "[OPTIONS] CONTAINER", "Return low-level information on a container") @@ -496,7 +504,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri for idx, field := range []string{ /* NAME */ name, /* ID */ img.Id, - /* CREATED */ future.HumanDuration(time.Now().Sub(time.Now())) + " ago", // FIXME: should be img.Created + /* CREATED */ future.HumanDuration(time.Now().Sub(time.Unix(img.Created, 0))) + " ago", /* PARENT */ img.Parent, } { if idx == 0 { From 18f4e7e196eb894765094c1291e1a1f6599f4266 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 12 Mar 2013 05:17:51 -0700 Subject: [PATCH 40/50] Put back the relative paths for dev purpose --- client/client.go | 4 ++-- docker/docker.go | 2 +- dockerd/dockerd.go | 4 ++-- fs/layers_test.go | 2 +- fs/store_test.go | 2 +- server/server.go | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/client.go b/client/client.go index 4c4ea1c5e..a277a4b18 100644 --- a/client/client.go +++ b/client/client.go @@ -1,8 +1,8 @@ package client import ( - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/rcli" + "../future" + "../rcli" "io" "io/ioutil" "log" diff --git a/docker/docker.go b/docker/docker.go index 73939abb6..df8a09443 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -1,7 +1,7 @@ package main import ( - "github.com/dotcloud/docker/client" + "../client" "flag" "log" "os" diff --git a/dockerd/dockerd.go b/dockerd/dockerd.go index 456dfe9d6..22ae8df8e 100644 --- a/dockerd/dockerd.go +++ b/dockerd/dockerd.go @@ -1,8 +1,8 @@ package main import ( - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/server" + ".." + "../server" "flag" "log" ) diff --git a/fs/layers_test.go b/fs/layers_test.go index 9f8353fdd..fa4926f54 100644 --- a/fs/layers_test.go +++ b/fs/layers_test.go @@ -1,7 +1,7 @@ package fs import ( - "github.com/dotcloud/docker/fake" + "../fake" "io/ioutil" "os" "testing" diff --git a/fs/store_test.go b/fs/store_test.go index 33bb16c92..d5715d1ba 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -1,7 +1,7 @@ package fs import ( - "github.com/dotcloud/docker/fake" + "../fake" "errors" "fmt" "io/ioutil" diff --git a/server/server.go b/server/server.go index 00ef9c496..32b2ce584 100644 --- a/server/server.go +++ b/server/server.go @@ -6,10 +6,10 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker" - "github.com/dotcloud/docker/fs" - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/rcli" + ".." + "../fs" + "../future" + "../rcli" "io" "net/http" "net/url" From 2664668388bd2b1d7012b863d4f8d426fda17daf Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 12 Mar 2013 05:23:13 -0700 Subject: [PATCH 41/50] Add parent when needed on store.Create, fix the mountpoint on store.layer. --- fs/store.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/store.go b/fs/store.go index ef275ecdd..fe6b08861 100644 --- a/fs/store.go +++ b/fs/store.go @@ -117,6 +117,9 @@ func (store *Store) Create(layerData Archive, parent *Image, pth, comment string Comment: comment, store: store, } + if parent != nil { + img.Parent = parent.Id + } // FIXME: we shouldn't have to pass os.Stderr to AddLayer()... // FIXME: Archive should contain compression info. For now we only support uncompressed. _, err := store.layers.AddLayer(img.Id, layerData) @@ -198,7 +201,7 @@ func (image *Image) layers() ([]string, error) { var err error currentImg := image for currentImg != nil { - if layer := image.store.layers.Get(image.Id); layer != "" { + if layer := image.store.layers.Get(currentImg.Id); layer != "" { list = append(list, layer) } else { return list, fmt.Errorf("Layer not found for image %s", image.Id) @@ -249,7 +252,6 @@ func (image *Image) Mount(root, rw string) (*Mountpoint, error) { if err != nil { return nil, err } - for _, layer := range layers { roBranches += fmt.Sprintf("%v=ro:", layer) } From f2dc079c05c073d63b10a3c2a2bc784d1e94e82c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 12 Mar 2013 05:35:47 -0700 Subject: [PATCH 42/50] Add TestCommitRun unit test --- container_test.go | 71 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/container_test.go b/container_test.go index a2c49006e..ea2164642 100644 --- a/container_test.go +++ b/container_test.go @@ -1,6 +1,7 @@ package docker import ( + "./fs" "fmt" "io" "io/ioutil" @@ -10,16 +11,16 @@ import ( "time" ) -func TestStart(t *testing.T) { +func TestCommitRun(t *testing.T) { docker, err := newTestDocker() if err != nil { t.Fatal(err) } defer nuke(docker) - container, err := docker.Create( - "start_test", - "ls", - []string{"-al"}, + container1, err := docker.Create( + "precommit_test", + "/bin/sh", + []string{"-c", "echo hello > /world"}, GetTestImage(docker), &Config{ Ram: 33554432, @@ -28,22 +29,62 @@ func TestStart(t *testing.T) { if err != nil { t.Fatal(err) } - defer docker.Destroy(container) + defer docker.Destroy(container1) - if container.State.Running { + if container1.State.Running { t.Errorf("Container shouldn't be running") } - if err := container.Start(); err != nil { + if err := container1.Run(); err != nil { t.Fatal(err) } - container.Wait() - if container.State.Running { + if container1.State.Running { t.Errorf("Container shouldn't be running") } - // We should be able to call Wait again - container.Wait() - if container.State.Running { - t.Errorf("Container shouldn't be running") + + // FIXME: freeze the container before copying it to avoid data corruption? + rwTar, err := fs.Tar(container1.Mountpoint.Rw, fs.Uncompressed) + if err != nil { + t.Error(err) + } + // Create a new image from the container's base layers + a new layer from container changes + parentImg, err := docker.Store.Get(container1.Image) + if err != nil { + t.Error(err) + } + + img, err := docker.Store.Create(rwTar, parentImg, "test_commitrun", "unit test commited image") + if err != nil { + t.Error(err) + } + + // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world + + container2, err := docker.Create( + "postcommit_test", + "cat", + []string{"/world"}, + img, + &Config{ + Ram: 33554432, + }, + ) + if err != nil { + t.Fatal(err) + } + defer docker.Destroy(container2) + + stdout, err := container2.StdoutPipe() + stderr, err := container2.StderrPipe() + if err := container2.Start(); err != nil { + t.Fatal(err) + } + container2.Wait() + output, err := ioutil.ReadAll(stdout) + output2, err := ioutil.ReadAll(stderr) + stdout.Close() + stderr.Close() + if string(output) != "hello\n" { + t.Fatalf("\nout: %s\nerr: %s\n", string(output), string(output2)) } } @@ -308,7 +349,7 @@ func TestUser(t *testing.T) { // Set a username container, err = docker.Create( "user_root", - "id", + "/bin/id", []string{}, GetTestImage(docker), &Config{ From 45a8bba15e95b789c12a371036aa1ad42357dc08 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 12 Mar 2013 05:36:37 -0700 Subject: [PATCH 43/50] gofmt --- container.go | 2 +- container_test.go | 6 +++--- fs/store.go | 2 +- server/server.go | 8 ++++---- state.go | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/container.go b/container.go index 62d7ce064..7a3970353 100644 --- a/container.go +++ b/container.go @@ -1,9 +1,9 @@ package docker import ( + "./fs" "encoding/json" "errors" - "./fs" "github.com/kr/pty" "io" "io/ioutil" diff --git a/container_test.go b/container_test.go index ea2164642..8801b0e76 100644 --- a/container_test.go +++ b/container_test.go @@ -65,9 +65,9 @@ func TestCommitRun(t *testing.T) { []string{"/world"}, img, &Config{ - Ram: 33554432, - }, - ) + Ram: 33554432, + }, + ) if err != nil { t.Fatal(err) } diff --git a/fs/store.go b/fs/store.go index fe6b08861..4a4b0ce80 100644 --- a/fs/store.go +++ b/fs/store.go @@ -1,10 +1,10 @@ package fs import ( + "../future" "database/sql" "errors" "fmt" - "../future" _ "github.com/mattn/go-sqlite3" "github.com/shykes/gorp" //Forked to implement CreateTablesOpts "io" diff --git a/server/server.go b/server/server.go index 32b2ce584..91e037ca0 100644 --- a/server/server.go +++ b/server/server.go @@ -1,15 +1,15 @@ package server import ( + ".." + "../fs" + "../future" + "../rcli" "bufio" "bytes" "encoding/json" "errors" "fmt" - ".." - "../fs" - "../future" - "../rcli" "io" "net/http" "net/url" diff --git a/state.go b/state.go index e88298ec4..6b753ce83 100644 --- a/state.go +++ b/state.go @@ -1,8 +1,8 @@ package docker import ( - "fmt" "./future" + "fmt" "sync" "time" ) From de50c9c4e8d0276890cfb521e5b57da58cf470aa Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 12 Mar 2013 05:57:19 -0700 Subject: [PATCH 44/50] 'docker run' works with paths as well as image IDs --- fs/store.go | 20 ++++++++++++++++++++ server/server.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/fs/store.go b/fs/store.go index b0fdadcc1..fab4924d3 100644 --- a/fs/store.go +++ b/fs/store.go @@ -101,6 +101,26 @@ func (store *Store) List(pth string) ([]*Image, error) { return store.imageList(images), nil } +func (store *Store) Find(pth string) (*Image, error) { + pth = path.Clean(pth) + img, err := store.Get(pth) + if err != nil { + return nil, err + } else if img != nil { + return img, nil + } + + images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc limit 1", pth) + if err != nil { + return nil, err + } else if len(images) < 1 { + return nil, nil + } + img = images[0].(*Image) + img.store = store + return img, nil +} + func (store *Store) Get(id string) (*Image, error) { img, err := store.orm.Get(Image{}, id) if img == nil { diff --git a/server/server.go b/server/server.go index cbec3edc0..d28aeaf7c 100644 --- a/server/server.go +++ b/server/server.go @@ -835,7 +835,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) cmdline = []string{"/bin/bash", "-i"} } // Find the image - img, err := srv.images.Get(name) + img, err := srv.images.Find(name) if err != nil { return err } else if img == nil { From 6908628b555a1583f079f04ed34f5d81ce917459 Mon Sep 17 00:00:00 2001 From: creack Date: Tue, 12 Mar 2013 08:06:22 -0700 Subject: [PATCH 45/50] Add *.orig to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fc4ca5da3..d55740421 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ docker/docker dockerd/dockerd .*.swp a.out +*.orig From d91d0da45310a4ab8d6124b70a1f541352149bdc Mon Sep 17 00:00:00 2001 From: creack Date: Tue, 12 Mar 2013 08:59:32 -0700 Subject: [PATCH 46/50] Fix merge issue --- server/server.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/server/server.go b/server/server.go index 94a08a9bf..4d03f5886 100644 --- a/server/server.go +++ b/server/server.go @@ -102,18 +102,25 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + images, _ := srv.images.Images() + var imgcount int + if images == nil { + imgcount = 0 + } else { + imgcount = len(images) + } cmd := rcli.Subcmd(stdout, "info", "", "Display system-wide information.") if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() > 1 { + if cmd.NArg() > 0 { cmd.Usage() return nil } fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", len(srv.containers.List()), VERSION, - len(srv.images.ById)) + imgcount) return nil } @@ -732,10 +739,17 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *fs.Image, ports []int, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *fs.Image, ports []int, user string, tty bool, openStdin bool, memory int64, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] container, err := srv.containers.Create(id, cmd, args, img, - &docker.Config{Hostname: id, Ports: ports, User: user, Tty: tty, OpenStdin: openStdin}) + &docker.Config{ + Hostname: id, + Ports: ports, + User: user, + Tty: tty, + OpenStdin: openStdin, + Memory: memory, + }) if err != nil { return nil, err } From 453d49573c91a1c49ffbc54d040145804e38d91e Mon Sep 17 00:00:00 2001 From: creack Date: Tue, 12 Mar 2013 11:59:27 -0700 Subject: [PATCH 47/50] Put back the github.com path for the import --- client/client.go | 4 ++-- container.go | 2 +- container_test.go | 2 +- docker.go | 2 +- docker/docker.go | 2 +- docker_test.go | 2 +- dockerd/dockerd.go | 4 ++-- fs/layers.go | 2 +- fs/layers_test.go | 2 +- fs/store.go | 2 +- fs/store_test.go | 2 +- server/server.go | 10 ++++------ state.go | 2 +- 13 files changed, 18 insertions(+), 20 deletions(-) diff --git a/client/client.go b/client/client.go index 073fe02b0..814aed2f1 100644 --- a/client/client.go +++ b/client/client.go @@ -1,8 +1,8 @@ package client import ( - "../future" - "../rcli" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "io" "io/ioutil" "log" diff --git a/container.go b/container.go index cda114907..c80129e2a 100644 --- a/container.go +++ b/container.go @@ -1,9 +1,9 @@ package docker import ( - "./fs" "encoding/json" "errors" + "github.com/dotcloud/docker/fs" "github.com/kr/pty" "io" "io/ioutil" diff --git a/container_test.go b/container_test.go index d896553be..18beeba25 100644 --- a/container_test.go +++ b/container_test.go @@ -1,9 +1,9 @@ package docker import ( - "./fs" "bufio" "fmt" + "github.com/dotcloud/docker/fs" "io" "io/ioutil" "math/rand" diff --git a/docker.go b/docker.go index 49c6d8324..8e8b19082 100644 --- a/docker.go +++ b/docker.go @@ -1,9 +1,9 @@ package docker import ( - "./fs" "container/list" "fmt" + "github.com/dotcloud/docker/fs" "io/ioutil" "log" "os" diff --git a/docker/docker.go b/docker/docker.go index df8a09443..fa9011def 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -1,8 +1,8 @@ package main import ( - "../client" "flag" + "github.com/dotcloud/docker/client" "log" "os" "path" diff --git a/docker_test.go b/docker_test.go index 5e75906c5..3f51e8d84 100644 --- a/docker_test.go +++ b/docker_test.go @@ -1,7 +1,7 @@ package docker import ( - "./fs" + "github.com/dotcloud/docker/fs" "io" "io/ioutil" "log" diff --git a/dockerd/dockerd.go b/dockerd/dockerd.go index 22ae8df8e..b2337d342 100644 --- a/dockerd/dockerd.go +++ b/dockerd/dockerd.go @@ -1,9 +1,9 @@ package main import ( - ".." - "../server" "flag" + "github.com/dotcloud/docker" + "github.com/dotcloud/docker/server" "log" ) diff --git a/fs/layers.go b/fs/layers.go index 11452fcf2..be699a8a0 100644 --- a/fs/layers.go +++ b/fs/layers.go @@ -1,9 +1,9 @@ package fs import ( - "../future" "errors" "fmt" + "github.com/dotcloud/docker/future" "io" "io/ioutil" "os" diff --git a/fs/layers_test.go b/fs/layers_test.go index fa4926f54..9f8353fdd 100644 --- a/fs/layers_test.go +++ b/fs/layers_test.go @@ -1,7 +1,7 @@ package fs import ( - "../fake" + "github.com/dotcloud/docker/fake" "io/ioutil" "os" "testing" diff --git a/fs/store.go b/fs/store.go index 4f4ed368e..36449a0fb 100644 --- a/fs/store.go +++ b/fs/store.go @@ -1,10 +1,10 @@ package fs import ( - "../future" "database/sql" "errors" "fmt" + "github.com/dotcloud/docker/future" _ "github.com/mattn/go-sqlite3" "github.com/shykes/gorp" //Forked to implement CreateTablesOpts "io" diff --git a/fs/store_test.go b/fs/store_test.go index d5715d1ba..f2cc06534 100644 --- a/fs/store_test.go +++ b/fs/store_test.go @@ -1,9 +1,9 @@ package fs import ( - "../fake" "errors" "fmt" + "github.com/dotcloud/docker/fake" "io/ioutil" "os" "testing" diff --git a/server/server.go b/server/server.go index 4d03f5886..343458e64 100644 --- a/server/server.go +++ b/server/server.go @@ -1,15 +1,15 @@ package server import ( - ".." - "../fs" - "../future" - "../rcli" "bufio" "bytes" "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker" + "github.com/dotcloud/docker/fs" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "io" "net/http" "net/url" @@ -60,8 +60,6 @@ func (srv *Server) Help() string { {"mirror", "(debug only) (No documentation available)"}, {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"ps", "List containers"}, - {"pull", "Download a new image from a remote location"}, - {"put", "Import a new image from a local archive"}, {"reset", "Reset changes to a container's filesystem"}, {"restart", "Restart a running container"}, {"rm", "Remove a container"}, diff --git a/state.go b/state.go index 6b753ce83..e864f304f 100644 --- a/state.go +++ b/state.go @@ -1,8 +1,8 @@ package docker import ( - "./future" "fmt" + "github.com/dotcloud/docker/future" "sync" "time" ) From 94baf2c5b9b2eb49b554d933f9997da072905d88 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Tue, 12 Mar 2013 12:05:04 -0700 Subject: [PATCH 48/50] upgraded kernel to 3.5.0-25 to fix a kernel bug --- Vagrantfile | 4 ++-- puppet/modules/docker/manifests/init.pp | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index dcc3e50bd..c823a3958 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,11 +7,11 @@ Vagrant::Config.run do |config| # please see the online documentation at vagrantup.com. # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = "quantal64" + config.vm.box = "quantal64_3.5.0-25" # The url from where the 'config.vm.box' box will be fetched if it # doesn't already exist on the user's system. - config.vm.box_url = "http://unworkable.org/~niallo/quantal64.box" + config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box" # Boot with a GUI so you can see the screen. (Default is headless) # config.vm.boot_mode = :gui diff --git a/puppet/modules/docker/manifests/init.pp b/puppet/modules/docker/manifests/init.pp index ed3dd62b6..38b40a8e3 100644 --- a/puppet/modules/docker/manifests/init.pp +++ b/puppet/modules/docker/manifests/init.pp @@ -7,7 +7,11 @@ class docker { Package { ensure => "installed" } - package { ["lxc", "debootstrap", "wget", "bsdtar", "git"]: } + package { ["lxc", "debootstrap", "wget", "bsdtar", "git", + "linux-image-3.5.0-25-generic", + "linux-image-extra-3.5.0-25-generic", + "virtualbox-guest-utils", + "linux-headers-3.5.0-25-generic"]: } notify { "docker_url = $docker_url": withpath => true } @@ -50,6 +54,11 @@ class docker { creates => "/usr/local/bin/dockerd" } + exec { "vbox-add" : + require => Package["linux-headers-3.5.0-25-generic"], + command => "/etc/init.d/vboxadd setup", + } + service { "dockerd" : ensure => "running", start => "/sbin/initctl start dockerd", From b2de17bf2cfdf90f347109452bcb8763138cff46 Mon Sep 17 00:00:00 2001 From: Louis Opter Date: Tue, 12 Mar 2013 12:12:40 -0700 Subject: [PATCH 49/50] Automatically remove the rcfile generated by docker -i from /tmp --- client/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/client.go b/client/client.go index 4c4ea1c5e..814aed2f1 100644 --- a/client/client.go +++ b/client/client.go @@ -111,6 +111,7 @@ func InteractiveMode(scripts ...string) error { if err != nil { return err } + defer os.Remove(rcfile.Name()) io.WriteString(rcfile, "enable -n help\n") os.Setenv("PATH", tmp+":"+os.Getenv("PATH")) os.Setenv("PS1", "\\h docker> ") From e20a74d2477ea3feb7a9734c15b645d081dbebdf Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Tue, 12 Mar 2013 17:34:15 -0700 Subject: [PATCH 50/50] fixed formatting issue --- server/server.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 3ac518069..53fc23d41 100644 --- a/server/server.go +++ b/server/server.go @@ -68,6 +68,7 @@ func (srv *Server) Help() string { {"stop", "Stop a running container"}, {"tar", "Stream the contents of a container as a tar archive"}, {"umount", "(debug only) Mount a container's filesystem"}, + {"version", "Show the docker version information"}, {"wait", "Block until a container stops, then print its exit code"}, {"web", "A web UI for docker"}, {"write", "Write the contents of standard input to a container's file"}, @@ -97,6 +98,12 @@ func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } +// 'docker version': show version information +func (srv *Server) CmdVersion(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + fmt.Fprintf(stdout, "Version:%s\n", VERSION) + return nil +} + // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "info", "", "Display system-wide information.") @@ -912,7 +919,7 @@ func New() (*Server, error) { return nil, err } srv := &Server{ - images: images, + images: images, containers: containers, } return srv, nil