From f9b4bfa59ba954c8e2d1a0f8cd42f0e75e4d46f3 Mon Sep 17 00:00:00 2001 From: unclejack Date: Sat, 16 Aug 2014 13:27:04 +0300 Subject: [PATCH 01/67] make http usage for registry explicit Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Conflicts: daemon/config.go daemon/daemon.go graph/pull.go graph/push.go graph/tags.go registry/registry.go registry/service.go --- daemon/config.go | 2 + daemon/daemon.go | 2 +- docs/sources/reference/commandline/cli.md | 3 +- graph/pull.go | 4 +- graph/push.go | 4 +- graph/tags.go | 24 ++++++----- registry/registry.go | 49 +++++++++++++++++++++++ registry/service.go | 2 +- 8 files changed, 74 insertions(+), 16 deletions(-) diff --git a/daemon/config.go b/daemon/config.go index 8780294ce..bae0c8cd2 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -31,6 +31,7 @@ type Config struct { BridgeIface string BridgeIP string FixedCIDR string + InsecureRegistries []string InterContainerCommunication bool GraphDriver string GraphOptions []string @@ -55,6 +56,7 @@ func (config *Config) InstallFlags() { flag.StringVar(&config.BridgeIP, []string{"#bip", "-bip"}, "", "Use this CIDR notation address for the network bridge's IP, not compatible with -b") flag.StringVar(&config.BridgeIface, []string{"b", "-bridge"}, "", "Attach containers to a pre-existing network bridge\nuse 'none' to disable container networking") flag.StringVar(&config.FixedCIDR, []string{"-fixed-cidr"}, "", "IPv4 subnet for fixed IPs (ex: 10.20.0.0/16)\nthis subnet must be nested in the bridge subnet (which is defined by -b or --bip)") + opts.ListVar(&config.InsecureRegistries, []string{"-insecure-registry"}, "Make these registries use http") flag.BoolVar(&config.InterContainerCommunication, []string{"#icc", "-icc"}, true, "Enable inter-container communication") flag.StringVar(&config.GraphDriver, []string{"s", "-storage-driver"}, "", "Force the Docker runtime to use a specific storage driver") flag.StringVar(&config.ExecDriver, []string{"e", "-exec-driver"}, "native", "Force the Docker runtime to use a specific exec driver") diff --git a/daemon/daemon.go b/daemon/daemon.go index 235788c68..d3c79c871 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -831,7 +831,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) } log.Debugf("Creating repository list") - repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors) + repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, config.Mirrors, config.InsecureRegistries) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 86f02b6cf..94cbb580b 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -70,7 +70,8 @@ expect an integer, and they can only be specified once. -g, --graph="/var/lib/docker" Path to use as the root of the Docker runtime -H, --host=[] The socket(s) to bind to in daemon mode or connect to in client mode, specified using one or more tcp://host:port, unix:///path/to/socket, fd://* or fd://socketfd. --icc=true Enable inter-container communication - --ip=0.0.0.0 Default IP address to use when binding container ports + --insecure-registry=[] Make these registries use http + --ip=0.0.0.0 Default IP address to use when binding container ports --ip-forward=true Enable net.ipv4.ip_forward --ip-masq=true Enable IP masquerading for bridge's IP range --iptables=true Enable Docker's addition of iptables rules diff --git a/graph/pull.go b/graph/pull.go index 5d7e84ed7..fe1237de4 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -113,7 +113,9 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Error(err) } - endpoint, err := registry.NewEndpoint(hostname) + secure := registry.IsSecure(hostname, s.InsecureRegistries) + + endpoint, err := registry.NewEndpoint(hostname, secure) if err != nil { return job.Error(err) } diff --git a/graph/push.go b/graph/push.go index 3511245b3..6b5b3394a 100644 --- a/graph/push.go +++ b/graph/push.go @@ -214,7 +214,9 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err) } - endpoint, err := registry.NewEndpoint(hostname) + secure := registry.IsSecure(hostname, s.InsecureRegistries) + + endpoint, err := registry.NewEndpoint(hostname, secure) if err != nil { return job.Error(err) } diff --git a/graph/tags.go b/graph/tags.go index 31c65ced5..c62aa8645 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -23,10 +23,11 @@ var ( ) type TagStore struct { - path string - graph *Graph - mirrors []string - Repositories map[string]Repository + path string + graph *Graph + mirrors []string + InsecureRegistries []string + Repositories map[string]Repository sync.Mutex // FIXME: move push/pull-related fields // to a helper type @@ -54,18 +55,19 @@ func (r Repository) Contains(u Repository) bool { return true } -func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) { +func NewTagStore(path string, graph *Graph, mirrors []string, insecureRegistries []string) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err } store := &TagStore{ - path: abspath, - graph: graph, - mirrors: mirrors, - Repositories: make(map[string]Repository), - pullingPool: make(map[string]chan struct{}), - pushingPool: make(map[string]chan struct{}), + path: abspath, + graph: graph, + mirrors: mirrors, + InsecureRegistries: insecureRegistries, + Repositories: make(map[string]Repository), + pullingPool: make(map[string]chan struct{}), + pushingPool: make(map[string]chan struct{}), } // Load the json file if it exists, otherwise create it. if err := store.reload(); os.IsNotExist(err) { diff --git a/registry/registry.go b/registry/registry.go index fd74b7514..8f4ae6fa0 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -202,6 +202,55 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } +// this method expands the registry name as used in the prefix of a repo +// to a full url. if it already is a url, there will be no change. +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { + if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { + // if there is no slash after https:// (8 characters) then we have no path in the url + if strings.LastIndex(hostname, "/") < 9 { + // there is no path given. Expand with default path + hostname = hostname + "/v1/" + } + if _, err := pingRegistryEndpoint(hostname); err != nil { + return "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + return hostname, nil + } + + // use HTTPS if secure, otherwise use HTTP + if secure { + endpoint = fmt.Sprintf("https://%s/v1/", hostname) + } else { + endpoint = fmt.Sprintf("http://%s/v1/", hostname) + } + _, err = pingRegistryEndpoint(endpoint) + if err != nil { + //TODO: triggering highland build can be done there without "failing" + err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + if secure { + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + } + return "", err + } + return endpoint, nil +} + +// this method verifies if the provided hostname is part of the list of +// insecure registries and returns false if HTTP should be used +func IsSecure(hostname string, insecureRegistries []string) (secure bool) { + secure = true + for _, h := range insecureRegistries { + if hostname == h { + secure = false + break + } + } + if hostname == IndexServerAddress() { + secure = true + } + return +} + func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/registry/service.go b/registry/service.go index f7b353000..334e7c2ed 100644 --- a/registry/service.go +++ b/registry/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr) + endpoint, err := NewEndpoint(addr, true) if err != nil { return job.Error(err) } From c0598aced053f7a7e06aebd57329348dbc7dfc10 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 19 Aug 2014 11:54:42 -0700 Subject: [PATCH 02/67] Refactor IsSecure change Fix issue with restoring the tag store and setting static configuration from the daemon. i.e. the field on the TagStore struct must be made internal or the json.Unmarshal in restore will overwrite the insecure registries to be an empty struct. Signed-off-by: Michael Crosby Conflicts: graph/pull.go graph/push.go graph/tags.go --- graph/pull.go | 2 +- graph/push.go | 2 +- graph/tags.go | 5 +++-- registry/registry.go | 44 +++++++++++++++++++------------------------- 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index fe1237de4..897e0813a 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -113,7 +113,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Error(err) } - secure := registry.IsSecure(hostname, s.InsecureRegistries) + secure := registry.IsSecure(hostname, s.insecureRegistries) endpoint, err := registry.NewEndpoint(hostname, secure) if err != nil { diff --git a/graph/push.go b/graph/push.go index 6b5b3394a..1d2bf7242 100644 --- a/graph/push.go +++ b/graph/push.go @@ -214,7 +214,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err) } - secure := registry.IsSecure(hostname, s.InsecureRegistries) + secure := registry.IsSecure(hostname, s.insecureRegistries) endpoint, err := registry.NewEndpoint(hostname, secure) if err != nil { diff --git a/graph/tags.go b/graph/tags.go index c62aa8645..622d62094 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -26,7 +26,7 @@ type TagStore struct { path string graph *Graph mirrors []string - InsecureRegistries []string + insecureRegistries []string Repositories map[string]Repository sync.Mutex // FIXME: move push/pull-related fields @@ -60,11 +60,12 @@ func NewTagStore(path string, graph *Graph, mirrors []string, insecureRegistries if err != nil { return nil, err } + store := &TagStore{ path: abspath, graph: graph, mirrors: mirrors, - InsecureRegistries: insecureRegistries, + insecureRegistries: insecureRegistries, Repositories: make(map[string]Repository), pullingPool: make(map[string]chan struct{}), pushingPool: make(map[string]chan struct{}), diff --git a/registry/registry.go b/registry/registry.go index 8f4ae6fa0..bcbce4019 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -204,51 +204,45 @@ func ResolveRepositoryName(reposName string) (string, string, error) { // this method expands the registry name as used in the prefix of a repo // to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { + if hostname == IndexServerAddress() { return hostname, nil } - // use HTTPS if secure, otherwise use HTTP + endpoint := fmt.Sprintf("http://%s/v1/", hostname) + if secure { endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } else { - endpoint = fmt.Sprintf("http://%s/v1/", hostname) } - _, err = pingRegistryEndpoint(endpoint) - if err != nil { + + if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { //TODO: triggering highland build can be done there without "failing" - err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) + if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) } + return "", err } + return endpoint, nil } // this method verifies if the provided hostname is part of the list of // insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) (secure bool) { - secure = true +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + for _, h := range insecureRegistries { if hostname == h { - secure = false - break + return false } } - if hostname == IndexServerAddress() { - secure = true - } - return + + return true } func trustedLocation(req *http.Request) bool { From c66196a9dc0cd7d19eb3535c52fdbccfa2ee628e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 19 Aug 2014 12:27:23 -0700 Subject: [PATCH 03/67] Expand documentation for --insecure-registries Signed-off-by: Michael Crosby --- docs/sources/reference/commandline/cli.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 94cbb580b..ddfc983d3 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -71,7 +71,7 @@ expect an integer, and they can only be specified once. -H, --host=[] The socket(s) to bind to in daemon mode or connect to in client mode, specified using one or more tcp://host:port, unix:///path/to/socket, fd://* or fd://socketfd. --icc=true Enable inter-container communication --insecure-registry=[] Make these registries use http - --ip=0.0.0.0 Default IP address to use when binding container ports + --ip=0.0.0.0 Default IP address to use when binding container ports --ip-forward=true Enable net.ipv4.ip_forward --ip-masq=true Enable IP masquerading for bridge's IP range --iptables=true Enable Docker's addition of iptables rules @@ -196,6 +196,16 @@ can be disabled with --ip-masq=false. +By default docker will assume all registries are securied via TLS. Prior versions +of docker used an auto fallback if a registry did not support TLS. This introduces +the opportunity for MITM attacks so in Docker 1.2 the user must specify `--insecure-registries` +when starting the Docker daemon to state which registries are not using TLS and to communicate +with these registries via plain text. If you are running a local registry over plain text +on `127.0.0.1:5000` you will be required to specify `--insecure-registries 127.0.0.1:500` +when starting the docker daemon to be able to push and pull images to that registry. +No automatic fallback will happen after Docker 1.2 to detect if a registry is using +HTTP or HTTPS. + Docker supports softlinks for the Docker data directory (`/var/lib/docker`) and for `/var/lib/docker/tmp`. The `DOCKER_TMPDIR` and the data directory can be set like this: From f43e77fc125840f40af52da01cd2a62bca49765e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 20 Aug 2014 08:31:24 -0700 Subject: [PATCH 04/67] Don't hard code true for auth job Signed-off-by: Michael Crosby Conflicts: registry/service.go --- builtins/builtins.go | 4 ++-- docker/daemon.go | 7 +++++++ registry/service.go | 22 +++++++++++----------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/builtins/builtins.go b/builtins/builtins.go index f952d728b..41bb24928 100644 --- a/builtins/builtins.go +++ b/builtins/builtins.go @@ -10,7 +10,6 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/events" "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/registry" ) func Register(eng *engine.Engine) error { @@ -26,7 +25,8 @@ func Register(eng *engine.Engine) error { if err := eng.Register("version", dockerVersion); err != nil { return err } - return registry.NewService().Install(eng) + + return nil } // remote: a RESTful api for cross-docker communication diff --git a/docker/daemon.go b/docker/daemon.go index eef17efdc..8b5826f34 100644 --- a/docker/daemon.go +++ b/docker/daemon.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/engine" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/signal" + "github.com/docker/docker/registry" ) const CanDaemon = true @@ -33,11 +34,17 @@ func mainDaemon() { } eng := engine.New() signal.Trap(eng.Shutdown) + // Load builtins if err := builtins.Register(eng); err != nil { log.Fatal(err) } + // load registry service + if err := registry.NewService(daemonCfg.InsecureRegistries).Install(eng); err != nil { + log.Fatal(err) + } + // load the daemon in the background so we can immediately start // the http api so that connections don't fail while the daemon // is booting diff --git a/registry/service.go b/registry/service.go index 334e7c2ed..890837ca5 100644 --- a/registry/service.go +++ b/registry/service.go @@ -13,12 +13,15 @@ import ( // 'pull': Download images from any registry (TODO) // 'push': Upload images to any registry (TODO) type Service struct { + insecureRegistries []string } // NewService returns a new instance of Service ready to be // installed no an engine. -func NewService() *Service { - return &Service{} +func NewService(insecureRegistries []string) *Service { + return &Service{ + insecureRegistries: insecureRegistries, + } } // Install installs registry capabilities to eng. @@ -32,15 +35,12 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var ( - err error - authConfig = &AuthConfig{} - ) + var authConfig = new(AuthConfig) job.GetenvJson("authConfig", authConfig) - // TODO: this is only done here because auth and registry need to be merged into one pkg + if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, true) + endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) if err != nil { return job.Error(err) } @@ -49,11 +49,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { } authConfig.ServerAddress = endpoint.String() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + + if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { return job.Error(err) } - job.Printf("%s\n", status) + return engine.StatusOK } From e134f1f74a9d1f9ec55fcd8af8c2dce8e9558618 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 10 Oct 2014 23:22:12 -0400 Subject: [PATCH 05/67] Do not verify certificate when using --insecure-registry on an HTTPS registry Signed-off-by: Tibor Vass Conflicts: registry/registry.go registry/registry_test.go registry/service.go registry/session.go --- daemon/config.go | 2 +- docs/sources/reference/commandline/cli.md | 20 +-- graph/tags_unit_test.go | 2 +- registry/endpoint.go | 47 +++++-- registry/registry.go | 143 +++++++++------------- registry/registry_test.go | 4 +- registry/service.go | 5 +- registry/session.go | 2 +- 8 files changed, 113 insertions(+), 112 deletions(-) diff --git a/daemon/config.go b/daemon/config.go index bae0c8cd2..9e8d08e2a 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -56,7 +56,7 @@ func (config *Config) InstallFlags() { flag.StringVar(&config.BridgeIP, []string{"#bip", "-bip"}, "", "Use this CIDR notation address for the network bridge's IP, not compatible with -b") flag.StringVar(&config.BridgeIface, []string{"b", "-bridge"}, "", "Attach containers to a pre-existing network bridge\nuse 'none' to disable container networking") flag.StringVar(&config.FixedCIDR, []string{"-fixed-cidr"}, "", "IPv4 subnet for fixed IPs (ex: 10.20.0.0/16)\nthis subnet must be nested in the bridge subnet (which is defined by -b or --bip)") - opts.ListVar(&config.InsecureRegistries, []string{"-insecure-registry"}, "Make these registries use http") + opts.ListVar(&config.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback)") flag.BoolVar(&config.InterContainerCommunication, []string{"#icc", "-icc"}, true, "Enable inter-container communication") flag.StringVar(&config.GraphDriver, []string{"s", "-storage-driver"}, "", "Force the Docker runtime to use a specific storage driver") flag.StringVar(&config.ExecDriver, []string{"e", "-exec-driver"}, "native", "Force the Docker runtime to use a specific exec driver") diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index ddfc983d3..4c863ee7b 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -70,7 +70,7 @@ expect an integer, and they can only be specified once. -g, --graph="/var/lib/docker" Path to use as the root of the Docker runtime -H, --host=[] The socket(s) to bind to in daemon mode or connect to in client mode, specified using one or more tcp://host:port, unix:///path/to/socket, fd://* or fd://socketfd. --icc=true Enable inter-container communication - --insecure-registry=[] Make these registries use http + --insecure-registry=[] Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) --ip=0.0.0.0 Default IP address to use when binding container ports --ip-forward=true Enable net.ipv4.ip_forward --ip-masq=true Enable IP masquerading for bridge's IP range @@ -195,16 +195,16 @@ to other machines on the Internet. This may interfere with some network topologi can be disabled with --ip-masq=false. +By default, Docker will assume all registries are secured via TLS with certificate verification +enabled. Prior versions of Docker used an auto fallback if a registry did not support TLS +(or if the TLS connection failed). This introduced the opportunity for Man In The Middle (MITM) +attacks, so as of Docker 1.3.1, the user must now specify the `--insecure-registry` daemon flag +for each insecure registry. An insecure registry is either not using TLS (i.e. plain text HTTP), +or is using TLS with a CA certificate not known by the Docker daemon (i.e. certification +verification disabled). For example, if there is a registry listening for HTTP at 127.0.0.1:5000, +as of Docker 1.3.1 you are required to specify `--insecure-registry 127.0.0.1:5000` when starting +the Docker daemon. -By default docker will assume all registries are securied via TLS. Prior versions -of docker used an auto fallback if a registry did not support TLS. This introduces -the opportunity for MITM attacks so in Docker 1.2 the user must specify `--insecure-registries` -when starting the Docker daemon to state which registries are not using TLS and to communicate -with these registries via plain text. If you are running a local registry over plain text -on `127.0.0.1:5000` you will be required to specify `--insecure-registries 127.0.0.1:500` -when starting the docker daemon to be able to push and pull images to that registry. -No automatic fallback will happen after Docker 1.2 to detect if a registry is using -HTTP or HTTPS. Docker supports softlinks for the Docker data directory (`/var/lib/docker`) and for `/var/lib/docker/tmp`. The `DOCKER_TMPDIR` and the data directory can be set like this: diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index e4f1fb809..da512547d 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -53,7 +53,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err != nil { t.Fatal(err) } - store, err := NewTagStore(path.Join(root, "tags"), graph, nil) + store, err := NewTagStore(path.Join(root, "tags"), graph, nil, nil) if err != nil { t.Fatal(err) } diff --git a/registry/endpoint.go b/registry/endpoint.go index 5313a8079..6dd4e1f60 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -2,7 +2,6 @@ package registry import ( "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -34,9 +33,9 @@ func scanForApiVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string) (*Endpoint, error) { +func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { var ( - endpoint Endpoint + endpoint = Endpoint{secure: secure} trimmedHostname string err error ) @@ -49,14 +48,27 @@ func NewEndpoint(hostname string) (*Endpoint, error) { return nil, err } + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - // TODO: Check if http fallback is enabled - endpoint.URL.Scheme = "http" - if _, err = endpoint.Ping(); err != nil { - return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + + //TODO: triggering highland build can be done there without "failing" + + if secure { + // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` + // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. + return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } + + // If registry is insecure and HTTPS failed, fallback to HTTP. + log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) + endpoint.URL.Scheme = "http" + _, err2 := endpoint.Ping() + if err2 == nil { + return &endpoint, nil + } + + return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return &endpoint, nil @@ -65,6 +77,7 @@ func NewEndpoint(hostname string) (*Endpoint, error) { type Endpoint struct { URL *url.URL Version APIVersion + secure bool } // Get the formated URL for the root of this registry Endpoint @@ -88,7 +101,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,3 +140,19 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + + for _, h := range insecureRegistries { + if hostname == h { + return false + } + } + + return true +} diff --git a/registry/registry.go b/registry/registry.go index bcbce4019..15fed1b8a 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) @@ -35,13 +36,17 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { tlsConfig := tls.Config{RootCAs: roots} if cert != nil { tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) } + if !secure { + tlsConfig.InsecureSkipVerify = true + } + httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -78,69 +83,76 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, } } -func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - - hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { - return nil, nil, err - } - +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { var ( pool *x509.CertPool certs []*tls.Certificate ) - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if pool == nil { - pool = x509.NewCertPool() + if secure && req.URL.Scheme == "https" { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } } - data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) - if err != nil { - return nil, nil, err - } - pool.AppendCertsFromPEM(data) + return false } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - if !hasFile(fs, keyName) { - return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return nil, nil, err - } - certs = append(certs, &cert) + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + log.Debugf("hostDir: %s", hostDir) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - if !hasFile(fs, certName) { - return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + log.Debugf("crt: %s", hostDir+"/"+f.Name()) + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } + pool.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + log.Debugf("cert: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + log.Debugf("key: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } } } } if len(certs) == 0 { - client := newClient(jar, pool, nil, timeout) + client := newClient(jar, pool, nil, timeout, secure) res, err := client.Do(req) if err != nil { return nil, nil, err } return res, client, nil } + for i, cert := range certs { - client := newClient(jar, pool, cert, timeout) + client := newClient(jar, pool, cert, timeout, secure) res, err := client.Do(req) // If this is the last cert, otherwise, continue to next cert if 403 or 5xx if i == len(certs)-1 || err == nil && res.StatusCode != 403 && res.StatusCode < 500 { @@ -202,49 +214,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { - if hostname == IndexServerAddress() { - return hostname, nil - } - - endpoint := fmt.Sprintf("http://%s/v1/", hostname) - - if secure { - endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } - - if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { - //TODO: triggering highland build can be done there without "failing" - err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) - - if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) - } - - return "", err - } - - return endpoint, nil -} - -// this method verifies if the provided hostname is part of the list of -// insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { - return true - } - - for _, h := range insecureRegistries { - if hostname == h { - return false - } - } - - return true -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/registry/registry_test.go b/registry/registry_test.go index ab4178126..c9a9fc81b 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -18,7 +18,7 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/")) + endpoint, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } @@ -30,7 +30,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } diff --git a/registry/service.go b/registry/service.go index 890837ca5..32274f407 100644 --- a/registry/service.go +++ b/registry/service.go @@ -89,7 +89,10 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - endpoint, err := NewEndpoint(hostname) + + secure := IsSecure(hostname, s.insecureRegistries) + + endpoint, err := NewEndpoint(hostname, secure) if err != nil { return job.Error(err) } diff --git a/registry/session.go b/registry/session.go index 5067b8d5d..28959967d 100644 --- a/registry/session.go +++ b/registry/session.go @@ -64,7 +64,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) } // Retrieve the history of a given image from the Registry. From ada9ac7b1300d2c647a17f5f5f274195ef5620b6 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Thu, 16 Oct 2014 11:39:22 -0700 Subject: [PATCH 06/67] Setting iptables=false should propagate to ip-masq=false Signed-off-by: Jessica Frazelle --- daemon/daemon.go | 2 +- integration-cli/docker_cli_daemon_test.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index d3c79c871..d069797ac 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -731,7 +731,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, fmt.Errorf("You specified --iptables=false with --icc=false. ICC uses iptables to function. Please set --icc or --iptables to true.") } if !config.EnableIptables && config.EnableIpMasq { - return nil, fmt.Errorf("You specified --iptables=false with --ipmasq=true. IP masquerading uses iptables to function. Please set --ipmasq to false or --iptables to true.") + config.EnableIpMasq = false } config.DisableNetwork = config.BridgeIface == disableNetworkBridge diff --git a/integration-cli/docker_cli_daemon_test.go b/integration-cli/docker_cli_daemon_test.go index 906680dc6..6160e57e9 100644 --- a/integration-cli/docker_cli_daemon_test.go +++ b/integration-cli/docker_cli_daemon_test.go @@ -82,3 +82,13 @@ func TestDaemonRestartWithVolumesRefs(t *testing.T) { logDone("daemon - volume refs are restored") } + +func TestDaemonStartIptablesFalse(t *testing.T) { + d := NewDaemon(t) + if err := d.Start("--iptables=false"); err != nil { + t.Fatalf("we should have been able to start the daemon with passing iptables=false: %v", err) + } + d.Stop() + + logDone("daemon - started daemon with iptables=false") +} From 7d9ccc2636af30e45950b27d3f69cacb0ad43f40 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 16 Oct 2014 10:14:26 -0600 Subject: [PATCH 07/67] Fix more missing HOME references Signed-off-by: Andrew Page --- hack/release.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/release.sh b/hack/release.sh index da832e8bb..1174e0cc4 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -270,7 +270,7 @@ EOF done # Upload keys - s3cmd sync /.gnupg/ s3://$BUCKET/ubuntu/.gnupg/ + s3cmd sync $HOME/.gnupg/ s3://$BUCKET/ubuntu/.gnupg/ gpg --armor --export releasedocker > bundles/$VERSION/ubuntu/gpg s3cmd --acl-public put bundles/$VERSION/ubuntu/gpg s3://$BUCKET/gpg @@ -355,8 +355,8 @@ release_test() { setup_gpg() { # Make sure that we have our keys - mkdir -p /.gnupg/ - s3cmd sync s3://$BUCKET/ubuntu/.gnupg/ /.gnupg/ || true + mkdir -p $HOME/.gnupg/ + s3cmd sync s3://$BUCKET/ubuntu/.gnupg/ $HOME/.gnupg/ || true gpg --list-keys releasedocker >/dev/null || { gpg --gen-key --batch < Date: Wed, 15 Oct 2014 22:39:51 -0400 Subject: [PATCH 08/67] Avoid fallback to SSL protocols < TLS1.0 Signed-off-by: Tibor Vass Docker-DCO-1.1-Signed-off-by: Daniel, Dao Quang Minh (github: dqminh) Conflicts: registry/registry.go --- api/server/server.go | 2 ++ docker/docker.go | 2 ++ registry/registry.go | 6 +++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/api/server/server.go b/api/server/server.go index 897dd6142..93b8b60a8 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1439,6 +1439,8 @@ func ListenAndServe(proto, addr string, job *engine.Job) error { tlsConfig := &tls.Config{ NextProtos: []string{"http/1.1"}, Certificates: []tls.Certificate{cert}, + // Avoid fallback on insecure SSL protocols + MinVersion: tls.VersionTLS10, } if job.GetenvBool("TlsVerify") { certPool := x509.NewCertPool() diff --git a/docker/docker.go b/docker/docker.go index 37cd155bb..f0cbb6f6a 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -93,6 +93,8 @@ func main() { } tlsConfig.Certificates = []tls.Certificate{cert} } + // Avoid fallback to SSL protocols < TLS1.0 + tlsConfig.MinVersion = tls.VersionTLS10 } if *flTls || *flTlsVerify { diff --git a/registry/registry.go b/registry/registry.go index 15fed1b8a..a03790af0 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -37,7 +37,11 @@ const ( ) func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { - tlsConfig := tls.Config{RootCAs: roots} + tlsConfig := tls.Config{ + RootCAs: roots, + // Avoid fallback to SSL protocols < TLS1.0 + MinVersion: tls.VersionTLS10, + } if cert != nil { tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) From cf23053eb1d88ce7b3ba2ca06f42c408021c93b5 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Fri, 17 Oct 2014 19:15:07 +0000 Subject: [PATCH 09/67] builder: fix escaping for ENV variables. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- builder/support.go | 14 +++++- integration-cli/docker_cli_build_test.go | 55 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/builder/support.go b/builder/support.go index a084190f2..6c7ac4096 100644 --- a/builder/support.go +++ b/builder/support.go @@ -10,13 +10,25 @@ var ( // `\$` - match literal $ // `[[:alnum:]_]+` - match things like `$SOME_VAR` // `{[[:alnum:]_]+}` - match things like `${SOME_VAR}` - tokenEnvInterpolation = regexp.MustCompile(`(\\\\+|[^\\]|\b|\A)\$([[:alnum:]_]+|{[[:alnum:]_]+})`) + tokenEnvInterpolation = regexp.MustCompile(`(\\|\\\\+|[^\\]|\b|\A)\$([[:alnum:]_]+|{[[:alnum:]_]+})`) // this intentionally punts on more exotic interpolations like ${SOME_VAR%suffix} and lets the shell handle those directly ) // handle environment replacement. Used in dispatcher. func (b *Builder) replaceEnv(str string) string { for _, match := range tokenEnvInterpolation.FindAllString(str, -1) { + idx := strings.Index(match, "\\$") + if idx != -1 { + if idx+2 >= len(match) { + str = strings.Replace(str, match, "\\$", -1) + continue + } + + stripped := match[idx+2:] + str = strings.Replace(str, match, "$"+stripped, -1) + continue + } + match = match[strings.Index(match, "$"):] matchKey := strings.Trim(match, "${}") diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index e456d579a..ac792df00 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -15,6 +15,61 @@ import ( "github.com/docker/docker/pkg/archive" ) +func TestBuildEnvEscapes(t *testing.T) { + name := "testbuildenvescapes" + defer deleteAllContainers() + defer deleteImages(name) + _, err := buildImage(name, + ` + FROM busybox + ENV TEST foo + CMD echo \$ + `, + true) + + out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "run", "-t", name)) + + if err != nil { + t.Fatal(err) + } + + if strings.TrimSpace(out) != "$" { + t.Fatalf("Env TEST was not overwritten with bar when foo was supplied to dockerfile: was %q", strings.TrimSpace(out)) + } + + logDone("build - env should handle \\$ properly") +} + +func TestBuildEnvOverwrite(t *testing.T) { + name := "testbuildenvoverwrite" + defer deleteAllContainers() + defer deleteImages(name) + + _, err := buildImage(name, + ` + FROM busybox + ENV TEST foo + CMD echo \${TEST} + `, + true) + + if err != nil { + t.Fatal(err) + } + + out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "run", "-e", "TEST=bar", "-t", name)) + + if err != nil { + t.Fatal(err) + } + + if strings.TrimSpace(out) != "bar" { + t.Fatalf("Env TEST was not overwritten with bar when foo was supplied to dockerfile: was %q", strings.TrimSpace(out)) + } + + logDone("build - env should overwrite builder ENV during run") +} + func TestBuildOnBuildForbiddenMaintainerInSourceImage(t *testing.T) { name := "testbuildonbuildforbiddenmaintainerinsourceimage" defer deleteImages(name) From ff325bcb2feb501c2287ecb305644b003784f726 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Fri, 17 Oct 2014 11:06:05 -0700 Subject: [PATCH 10/67] Don't write pull output to stdout on container creating Fixes #8632 Signed-off-by: Alexandr Morozov --- api/client/commands.go | 9 +++++++-- integration-cli/docker_cli_run_test.go | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/api/client/commands.go b/api/client/commands.go index 6c4e5c55f..2c44bb63c 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -1986,6 +1986,10 @@ func (cli *DockerCli) CmdTag(args ...string) error { } func (cli *DockerCli) pullImage(image string) error { + return cli.pullImageCustomOut(image, cli.out) +} + +func (cli *DockerCli) pullImageCustomOut(image string, out io.Writer) error { v := url.Values{} repos, tag := parsers.ParseRepositoryTag(image) // pull only the image tagged 'latest' if no tag was specified @@ -2014,7 +2018,7 @@ func (cli *DockerCli) pullImage(image string) error { registryAuthHeader := []string{ base64.URLEncoding.EncodeToString(buf), } - if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.out, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil { + if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, out, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil { return err } return nil @@ -2081,7 +2085,8 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc if statusCode == 404 { fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", config.Image) - if err = cli.pullImage(config.Image); err != nil { + // we don't want to write to stdout anything apart from container.ID + if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil { return nil, err } // Retry diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index d50f6f344..83f03fd9f 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "fmt" "io/ioutil" "net" @@ -2374,3 +2375,18 @@ func TestRunVolumesNotRecreatedOnStart(t *testing.T) { logDone("run - volumes not recreated on start") } + +func TestRunNoOutputFromPullInStdout(t *testing.T) { + defer deleteAllContainers() + // just run with unknown image + cmd := exec.Command(dockerBinary, "run", "asdfsg") + stdout := bytes.NewBuffer(nil) + cmd.Stdout = stdout + if err := cmd.Run(); err == nil { + t.Fatal("Run with unknown image should fail") + } + if stdout.Len() != 0 { + t.Fatalf("Stdout contains output from pull: %s", stdout) + } + logDone("run - no output from pull in stdout") +} From 66fba7c46ed3507166774ec8000ca478907e80a7 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 20 Oct 2014 15:27:26 -0400 Subject: [PATCH 11/67] Clean volume paths Fixes #8659 Signed-off-by: Brian Goff --- daemon/volumes.go | 3 ++ integration-cli/docker_cli_run_test.go | 50 ++++++++++++++++++++++++++ integration-cli/docker_test_vars.go | 6 ++-- integration-cli/docker_utils.go | 10 ++++++ volumes/repository.go | 5 +-- 5 files changed, 70 insertions(+), 4 deletions(-) diff --git a/daemon/volumes.go b/daemon/volumes.go index c7a8d7bfc..b34d9678c 100644 --- a/daemon/volumes.go +++ b/daemon/volumes.go @@ -133,6 +133,7 @@ func (container *Container) parseVolumeMountConfig() (map[string]*Mount, error) // Get the rest of the volumes for path := range container.Config.Volumes { // Check if this is already added as a bind-mount + path = filepath.Clean(path) if _, exists := mounts[path]; exists { continue } @@ -182,6 +183,8 @@ func parseBindMountSpec(spec string) (string, string, bool, error) { return "", "", false, fmt.Errorf("cannot bind mount volume: %s volume paths must be absolute.", path) } + path = filepath.Clean(path) + mountToPath = filepath.Clean(mountToPath) return path, mountToPath, writable, nil } diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index 83f03fd9f..417368f4f 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -2390,3 +2390,53 @@ func TestRunNoOutputFromPullInStdout(t *testing.T) { } logDone("run - no output from pull in stdout") } + +func TestRunVolumesCleanPaths(t *testing.T) { + defer deleteAllContainers() + + if _, err := buildImage("run_volumes_clean_paths", + `FROM busybox + VOLUME /foo/`, + true); err != nil { + t.Fatal(err) + } + defer deleteImages("run_volumes_clean_paths") + + cmd := exec.Command(dockerBinary, "run", "-v", "/foo", "-v", "/bar/", "--name", "dark_helmet", "run_volumes_clean_paths") + if out, _, err := runCommandWithOutput(cmd); err != nil { + t.Fatal(err, out) + } + + out, err := inspectFieldMap("dark_helmet", "Volumes", "/foo/") + if err != nil { + t.Fatal(err) + } + if out != "" { + t.Fatalf("Found unexpected volume entry for '/foo/' in volumes\n%q", out) + } + + out, err = inspectFieldMap("dark_helmet", "Volumes", "/foo") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, volumesStoragePath) { + t.Fatalf("Volume was not defined for /foo\n%q", out) + } + + out, err = inspectFieldMap("dark_helmet", "Volumes", "/bar/") + if err != nil { + t.Fatal(err) + } + if out != "" { + t.Fatalf("Found unexpected volume entry for '/bar/' in volumes\n%q", out) + } + out, err = inspectFieldMap("dark_helmet", "Volumes", "/bar") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, volumesStoragePath) { + t.Fatalf("Volume was not defined for /bar\n%q", out) + } + + logDone("run - volume paths are cleaned") +} diff --git a/integration-cli/docker_test_vars.go b/integration-cli/docker_test_vars.go index fdbcf073e..23903a39a 100644 --- a/integration-cli/docker_test_vars.go +++ b/integration-cli/docker_test_vars.go @@ -16,8 +16,10 @@ var ( // the private registry to use for tests privateRegistryURL = "127.0.0.1:5000" - execDriverPath = "/var/lib/docker/execdriver/native" - volumesConfigPath = "/var/lib/docker/volumes" + dockerBasePath = "/var/lib/docker" + execDriverPath = dockerBasePath + "/execdriver/native" + volumesConfigPath = dockerBasePath + "/volumes" + volumesStoragePath = dockerBasePath + "/vfs/dir" workingDirectory string ) diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index 17370eb70..3bdf36ec1 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -507,6 +507,16 @@ func inspectFieldJSON(name, field string) (string, error) { return strings.TrimSpace(out), nil } +func inspectFieldMap(name, path, field string) (string, error) { + format := fmt.Sprintf("{{index .%s %q}}", path, field) + inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name) + out, exitCode, err := runCommandWithOutput(inspectCmd) + if err != nil || exitCode != 0 { + return "", fmt.Errorf("failed to inspect %s: %s", name, out) + } + return strings.TrimSpace(out), nil +} + func getIDByName(name string) (string, error) { return inspectField(name, "Id") } diff --git a/volumes/repository.go b/volumes/repository.go index e765d944c..2383f34a9 100644 --- a/volumes/repository.go +++ b/volumes/repository.go @@ -55,6 +55,7 @@ func (r *Repository) newVolume(path string, writable bool) (*Volume, error) { return nil, err } } + path = filepath.Clean(path) path, err = filepath.EvalSymlinks(path) if err != nil { @@ -126,7 +127,7 @@ func (r *Repository) get(path string) *Volume { if err != nil { return nil } - return r.volumes[path] + return r.volumes[filepath.Clean(path)] } func (r *Repository) Add(volume *Volume) error { @@ -160,7 +161,7 @@ func (r *Repository) Delete(path string) error { if err != nil { return err } - volume := r.get(path) + volume := r.get(filepath.Clean(path)) if volume == nil { return fmt.Errorf("Volume %s does not exist", path) } From 21ab75afe0f7e92ffe942e30c77651a01b6db223 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Tue, 21 Oct 2014 19:26:20 +0000 Subject: [PATCH 12/67] builder: handle cases where onbuild is not uppercase. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- builder/dispatchers.go | 3 +- integration-cli/docker_cli_build_test.go | 35 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/builder/dispatchers.go b/builder/dispatchers.go index 82bb6ce5f..0c2a58087 100644 --- a/builder/dispatchers.go +++ b/builder/dispatchers.go @@ -11,6 +11,7 @@ import ( "fmt" "io/ioutil" "path/filepath" + "regexp" "strings" "github.com/docker/docker/nat" @@ -129,7 +130,7 @@ func onbuild(b *Builder, args []string, attributes map[string]bool, original str return fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction) } - original = strings.TrimSpace(strings.TrimLeft(original, "ONBUILD")) + original = regexp.MustCompile(`(?i)^\s*ONBUILD\s*`).ReplaceAllString(original, "") b.Config.OnBuild = append(b.Config.OnBuild, original) return b.commit("", b.Config.Cmd, fmt.Sprintf("ONBUILD %s", original)) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index ac792df00..808f64c7e 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -15,6 +15,41 @@ import ( "github.com/docker/docker/pkg/archive" ) +func TestBuildOnBuildLowercase(t *testing.T) { + name := "testbuildonbuildlowercase" + name2 := "testbuildonbuildlowercase2" + + defer deleteImages(name, name2) + + _, err := buildImage(name, + ` + FROM busybox + onbuild run echo quux + `, true) + + if err != nil { + t.Fatal(err) + } + + _, out, err := buildImageWithOut(name2, fmt.Sprintf(` + FROM %s + `, name), true) + + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(out, "quux") { + t.Fatalf("Did not receive the expected echo text, got %s", out) + } + + if strings.Contains(out, "ONBUILD ONBUILD") { + t.Fatalf("Got an ONBUILD ONBUILD error with no error: got %s", out) + } + + logDone("build - handle case-insensitive onbuild statement") +} + func TestBuildEnvEscapes(t *testing.T) { name := "testbuildenvescapes" defer deleteAllContainers() From 3d287811d78c373a4fd3ef069a9c13cd83b80169 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 22 Oct 2014 08:30:48 -0400 Subject: [PATCH 13/67] Docs edits for dropping SSLv3 and under + release notes for 1.3.1 Signed-off-by: Tibor Vass Conflicts: docs/sources/index.md --- docs/mkdocs.yml | 1 + docs/sources/index.md | 38 +-- .../reference/api/hub_registry_spec.md | 30 +- docs/sources/reference/api/registry_api.md | 29 +- docs/sources/reference/commandline/cli.md | 7 +- docs/sources/release-notes.md | 291 ++++++++++++++++++ 6 files changed, 334 insertions(+), 62 deletions(-) mode change 100755 => 100644 docs/mkdocs.yml create mode 100644 docs/sources/release-notes.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml old mode 100755 new mode 100644 index 5ea8d56e6..25f84b5a0 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -26,6 +26,7 @@ pages: # Introduction: - ['index.md', 'About', 'Docker'] +- ['release-notes.md', 'About', 'Release Notes'] - ['introduction/index.md', '**HIDDEN**'] - ['introduction/understanding-docker.md', 'About', 'Understanding Docker'] diff --git a/docs/sources/index.md b/docs/sources/index.md index 7e60b0dc6..949fe72e0 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -88,40 +88,4 @@ implementation, check out the [Docker User Guide](/userguide/). ## Release Notes -**Version 1.3.0** - -This version fixes a number of bugs and issues and adds new functions and other -improvements. These include: - -*New command: `docker exec`* - -The new `docker exec` command lets you run a process in an existing, active -container. The command has APIs for both the daemon and the client. With -`docker exec`, you'll be able to do things like add or remove devices from running containers, debug running containers, and run commands that are not -part of the container's static specification. - -*New command: `docker create`* - -Traditionally, the `docker run` command has been used to both create a -container and spawn a process to run it. The new `docker create` command breaks -this apart, letting you set up a container without actually starting it. This -provides more control over management of the container lifecycle, giving you the -ability to configure things like volumes or port mappings before the container -is started. For example, in a rapid-response scaling situation, you could use -`create` to prepare and stage ten containers in anticipation of heavy loads. - -*New provenance features* - -Official images are now signed by Docker, Inc. to improve your confidence and -security. Look for the blue ribbons on the [Docker Hub](https://hub.docker.com/). -The Docker Engine has been updated to automatically verify that a given Official -Repo has a current, valid signature. If no valid signature is detected, Docker -Engine will use a prior image. - - -*Other improvements & changes* - -We've added a new security options flag that lets you set SELinux and AppArmor -labels and profiles. This means you'll longer have to use `docker run ---privileged on kernels that support SE Linux or AppArmor. - +A summary of the changes in each release in the current series can now be found on the separate [Release Notes page](/release-notes/) diff --git a/docs/sources/reference/api/hub_registry_spec.md b/docs/sources/reference/api/hub_registry_spec.md index ee15277a4..853eda4ae 100644 --- a/docs/sources/reference/api/hub_registry_spec.md +++ b/docs/sources/reference/api/hub_registry_spec.md @@ -4,7 +4,9 @@ page_keywords: docker, registry, api, hub # The Docker Hub and the Registry spec -## The 3 roles +## The three roles + +There are three major components playing a role in the Docker ecosystem. ### Docker Hub @@ -21,13 +23,15 @@ The Docker Hub has different components: - Authentication service - Tokenization -The Docker Hub is authoritative for those information. +The Docker Hub is authoritative for that information. -We expect that there will be only one instance of the Docker Hub, run and +There is only one instance of the Docker Hub, run and managed by Docker Inc. ### Registry +The registry has the following characteristics: + - It stores the images and the graph for a set of repositories - It does not have user accounts data - It has no notion of user accounts or authorization @@ -37,35 +41,35 @@ managed by Docker Inc. - It doesn't have a local database - [Source Code](https://github.com/docker/docker-registry) -We expect that there will be multiple registries out there. To help to +We expect that there will be multiple registries out there. To help you grasp the context, here are some examples of registries: - **sponsor registry**: such a registry is provided by a third-party hosting infrastructure as a convenience for their customers and the - docker community as a whole. Its costs are supported by the third + Docker community as a whole. Its costs are supported by the third party, but the management and operation of the registry are - supported by dotCloud. It features read/write access, and delegates + supported by Docker, Inc. It features read/write access, and delegates authentication and authorization to the Docker Hub. - **mirror registry**: such a registry is provided by a third-party hosting infrastructure but is targeted at their customers only. Some mechanism (unspecified to date) ensures that public images are pulled from a sponsor registry to the mirror registry, to make sure - that the customers of the third-party provider can “docker pull” + that the customers of the third-party provider can `docker pull` those images locally. - **vendor registry**: such a registry is provided by a software - vendor, who wants to distribute docker images. It would be operated + vendor who wants to distribute docker images. It would be operated and managed by the vendor. Only users authorized by the vendor would be able to get write access. Some images would be public (accessible for anyone), others private (accessible only for authorized users). Authentication and authorization would be delegated to the Docker Hub. - The goal of vendor registries is to let someone do “docker pull - basho/riak1.3” and automatically push from the vendor registry - (instead of a sponsor registry); i.e. get all the convenience of a + The goal of vendor registries is to let someone do `docker pull + basho/riak1.3` and automatically push from the vendor registry + (instead of a sponsor registry); i.e., vendors get all the convenience of a sponsor registry, while retaining control on the asset distribution. - **private registry**: such a registry is located behind a firewall, or protected by an additional security layer (HTTP authorization, SSL client-side certificates, IP address authorization...). The - registry is operated by a private entity, outside of dotCloud's + registry is operated by a private entity, outside of Docker's control. It can optionally delegate additional authorization to the Docker Hub, but it is not mandatory. @@ -77,7 +81,7 @@ grasp the context, here are some examples of registries: > - local mount point; > - remote docker addressed through SSH. -The latter would only require two new commands in docker, e.g., +The latter would only require two new commands in Docker, e.g., `registryget` and `registryput`, wrapping access to the local filesystem (and optionally doing consistency checks). Authentication and authorization are then delegated diff --git a/docs/sources/reference/api/registry_api.md b/docs/sources/reference/api/registry_api.md index 1ae37dba6..d6130bf76 100644 --- a/docs/sources/reference/api/registry_api.md +++ b/docs/sources/reference/api/registry_api.md @@ -21,30 +21,30 @@ grasp the context, here are some examples of registries: - **sponsor registry**: such a registry is provided by a third-party hosting infrastructure as a convenience for their customers and the - docker community as a whole. Its costs are supported by the third + Docker community as a whole. Its costs are supported by the third party, but the management and operation of the registry are - supported by dotCloud. It features read/write access, and delegates + supported by Docker. It features read/write access, and delegates authentication and authorization to the Index. - **mirror registry**: such a registry is provided by a third-party hosting infrastructure but is targeted at their customers only. Some mechanism (unspecified to date) ensures that public images are pulled from a sponsor registry to the mirror registry, to make sure - that the customers of the third-party provider can “docker pull” + that the customers of the third-party provider can `docker pull` those images locally. - **vendor registry**: such a registry is provided by a software - vendor, who wants to distribute docker images. It would be operated + vendor, who wants to distribute Docker images. It would be operated and managed by the vendor. Only users authorized by the vendor would be able to get write access. Some images would be public (accessible for anyone), others private (accessible only for authorized users). Authentication and authorization would be delegated to the Index. - The goal of vendor registries is to let someone do “docker pull - basho/riak1.3” and automatically push from the vendor registry - (instead of a sponsor registry); i.e. get all the convenience of a + The goal of vendor registries is to let someone do `docker pull + basho/riak1.3` and automatically push from the vendor registry + (instead of a sponsor registry); i.e., get all the convenience of a sponsor registry, while retaining control on the asset distribution. - **private registry**: such a registry is located behind a firewall, or protected by an additional security layer (HTTP authorization, SSL client-side certificates, IP address authorization...). The - registry is operated by a private entity, outside of dotCloud's + registry is operated by a private entity, outside of Docker's control. It can optionally delegate additional authorization to the Index, but it is not mandatory. @@ -52,7 +52,7 @@ grasp the context, here are some examples of registries: > Mirror registries and private registries which do not use the Index > don't even need to run the registry code. They can be implemented by any > kind of transport implementing HTTP GET and PUT. Read-only registries -> can be powered by a simple static HTTP server. +> can be powered by a simple static HTTPS server. > **Note**: > The latter implies that while HTTP is the protocol of choice for a registry, @@ -60,13 +60,20 @@ grasp the context, here are some examples of registries: > > - HTTP with GET (and PUT for read-write registries); > - local mount point; -> - remote docker addressed through SSH. +> - remote Docker addressed through SSH. -The latter would only require two new commands in docker, e.g., +The latter would only require two new commands in Docker, e.g., `registryget` and `registryput`, wrapping access to the local filesystem (and optionally doing consistency checks). Authentication and authorization are then delegated to SSH (e.g., with public keys). +> **Note**: +> Private registry servers that expose an HTTP endpoint need to be secured with +> TLS (preferably TLSv1.2, but at least TLSv1.0). Make sure to put the CA +> certificate at /etc/docker/certs.d/my.registry.com:5000/ca.crt on the Docker +> host, so that the daemon can securely access the private registry. +> Support for SSLv3 and lower is not available due to security issues. + The default namespace for a private repository is `library`. # Endpoints diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 4c863ee7b..a57b4cc28 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -112,7 +112,12 @@ direct access to the Docker daemon - and should be secured either using the [built in https encrypted socket](/articles/https/), or by putting a secure web proxy in front of it. You can listen on port `2375` on all network interfaces with `-H tcp://0.0.0.0:2375`, or on a particular network interface using its IP -address: `-H tcp://192.168.59.103:2375`. +address: `-H tcp://192.168.59.103:2375`. It is conventional to use port `2375` +for un-encrypted, and port `2376` for encrypted communication with the daemon. + +> **Note** If you're using an HTTPS encrypted socket, keep in mind that only TLS1.0 +> and greater are supported. Protocols SSLv3 and under are not supported anymore +> for security reasons. On Systemd based systems, you can communicate with the daemon via [systemd socket activation](http://0pointer.de/blog/projects/socket-activation.html), use diff --git a/docs/sources/release-notes.md b/docs/sources/release-notes.md new file mode 100644 index 000000000..6b2f0fd57 --- /dev/null +++ b/docs/sources/release-notes.md @@ -0,0 +1,291 @@ +page_title: Docker 1.x Series Release Notes page_description: Release Notes for +Docker 1.x. page_keywords: docker, documentation, about, technology, +understanding, release + +#Release Notes + +##Version 1.3.1 +(2014-10-28) + +This release fixes some bugs and addresses some security issues. + +*Security fixes* + +Patches and changes were made to address CVE-2014-5277 and CVE-2014-3566. Specifically, changes were made to: +* Prevent fallback to SSL protocols < TLS 1.0 for client, daemon and registry +* Secure HTTPS connection to registries with certificate verification and without HTTP fallback unless `--insecure-registry` is specified. + +*Runtime fixes* + +* Fixed issue where volumes would not be shared + +*Client fixes* + +* Fixed issue with `--iptables=false` not automatically setting +`--ip-masq=false` +* Fixed docker run output to non-TTY stdout + +*Builder fixes* + +* Fixed escaping `$` for environment variables +* Fixed issue with lowercase `onbuild` Dockerfile instruction + + +##Version 1.3.0 + +This version fixes a number of bugs and issues and adds new functions and other +improvements. The [GitHub 1.3milestone](https://github.com/docker/docker/issues?q=milestone%3A1.3.0+) has +more detailed information. Major additions and changes include: + +###New Features + +*New command: `docker exec`* + +The new `docker exec` command lets you run a process in an existing, active +container. The command has APIs for both the daemon and the client. With `docker +exec`, you'll be able to do things like add or remove devices from running +containers, debug running containers, and run commands that are not part of the +container's static specification. Details in the [command line reference](/reference/commandline/cli/#exec). + +*New command: `docker create`* + +Traditionally, the `docker run` command has been used to both create a container +and spawn a process to run it. The new `docker create` command breaks this +apart, letting you set up a container without actually starting it. This +provides more control over management of the container lifecycle, giving you the +ability to configure things like volumes or port mappings before the container +is started. For example, in a rapid-response scaling situation, you could use +`create` to prepare and stage ten containers in anticipation of heavy loads. +Details in the [command line reference](/reference/commandline/cli/#create). + +*Tech preview of new provenance features* + +This release offers a sneak peek at new image signing capabilities that are +currently under development. Soon, these capabilities will allow any image +author to sign their images to certify they have not been tampered with. For +this release, Official images are now signed by Docker, Inc. Not only does this +demonstrate the new functionality, we hope it will improve your confidence in +the security of Official images. Look for the blue ribbons denoting signed +images on the [Docker Hub](https://hub.docker.com/). The Docker Engine has been +updated to automatically verify that a given Official Repo has a current, valid +signature. When pulling a signed image, you'll see a message stating `the image +you are pulling has been verified`. If no valid signature is detected, Docker +Engine will fall back to pulling a regular, unsigned image. + +###Other improvements & changes* + +* We've added a new security options flag to the `docker run` command, +`--security-opt`, that lets you set SELinux and AppArmor labels and profiles. +This means you'll no longer have to use `docker run --privileged` on kernels +that support SE Linux or AppArmor. For more information, see the [command line +reference](/reference/commandline/cli/#run). + +* A new flag, `--add-host`, has been added to `docker run` that lets you add +lines to `/etc/hosts`. This allows you to specify different name resolution for +the container than it would get via DNS. For more information, see the [command +line reference](/reference/commandline/cli/#run). + +* You can now set a `DOCKER_TLS_VERIFY` environment variable to secure +connections by default (rather than having to pass the `--tlsverify` flag on +every call). For more information, see the [https guide](/articles/https). + +* Three security issues have been addressed in this release: [CVE-2014-5280, +CVE-2014-5270, and +CVE-2014-5282](https://groups.google.com/forum/#!msg/docker-announce/aQoVmQlcE0A/smPuBNYf8VwJ). + +##Version 1.2.0 + +This version fixes a number of bugs and issues and adds new functions and other +improvements. These include: + +###New Features + +*New restart policies* + +We added a `--restart flag` to `docker run` to specify a restart policy for your +container. Currently, there are three policies available: + +* `no` – Do not restart the container if it dies. (default) * `on-failure` – +Restart the container if it exits with a non-zero exit code. This can also +accept an optional maximum restart count (e.g. `on-failure:5`). * `always` – +Always restart the container no matter what exit code is returned. This +deprecates the `--restart` flag on the Docker daemon. + +*New flags for `docker run`: `--cap-add` and `–-cap-drop`* + +In previous releases, Docker containers could either be given complete +capabilities or they could all follow a whitelist of allowed capabilities while +dropping all others. Further, using `--privileged` would grant all capabilities +inside a container, rather than applying a whitelist. This was not recommended +for production use because it’s really unsafe; it’s as if you were directly in +the host. + +This release introduces two new flags for `docker run`, `--cap-add` and +`--cap-drop`, that give you fine-grain control over the specific capabilities +you want grant to a particular container. + +*New `-–device` flag for `docker run`* + +Previously, you could only use devices inside your containers by bind mounting +them (with `-v`) in a `--privileged` container. With this release, we introduce +the `--device flag` to `docker run` which lets you use a device without +requiring a privileged container. + +*Writable `/etc/hosts`, `/etc/hostname` and `/etc/resolv.conf`* + +You can now edit `/etc/hosts`, `/etc/hostname` and `/etc/resolve.conf` in a +running container. This is useful if you need to install BIND or other services +that might override one of those files. + +Note, however, that changes to these files are not saved when running `docker +build` and so will not be preserved in the resulting image. The changes will +only “stick” in a running container. + +*Docker proxy in a separate process* + +The Docker userland proxy that routes outbound traffic to your containers now +has its own separate process (one process per connection). This greatly reduces +the load on the daemon, which increases stability and efficiency. + +###Other improvements & changes + +* When using `docker rm -f`, Docker now kills the container (instead of stopping +it) before removing it . If you intend to stop the container cleanly, you can +use `docker stop`. + +* Added support for IPv6 addresses in `--dns` + +* Added search capability in private registries + +##Version 1.1.0 + +###New Features + +*`.dockerignore` support* + +You can now add a `.dockerignore` file next to your `Dockerfile` and Docker will +ignore files and directories specified in that file when sending the build +context to the daemon. Example: +https://github.com/docker/docker/blob/master/.dockerignore + +*Pause containers during commit* + +Doing a commit on a running container was not recommended because you could end +up with files in an inconsistent state (for example, if they were being written +during the commit). Containers are now paused when a commit is made to them. You +can disable this feature by doing a `docker commit --pause=false ` + +*Tailing logs* + +You can now tail the logs of a container. For example, you can get the last ten +lines of a log by using `docker logs --tail 10 `. You can also +follow the logs of a container without having to read the whole log file with +`docker logs --tail 0 -f `. + +*Allow a tar file as context for docker build* + +You can now pass a tar archive to `docker build` as context. This can be used to +automate docker builds, for example: `cat context.tar | docker build -` or +`docker run builder_image | docker build -` + +*Bind mounting your whole filesystem in a container* + +`/` is now allowed as source of `--volumes`. This means you can bind-mount your +whole system in a container if you need to. For example: `docker run -v +/:/my_host ubuntu:ro ls /my_host`. However, it is now forbidden to mount to /. + + +###Other Improvements & Changes + +* Port allocation has been improved. In the previous release, Docker could +prevent you from starting a container with previously allocated ports which +seemed to be in use when in fact they were not. This has been fixed. + +* A bug in `docker save` was introduced in the last release. The `docker save` +command could produce images with invalid metadata. The command now produces +images with correct metadata. + +* Running `docker inspect` in a container now returns which containers it is +linked to. + +* Parsing of the `docker commit` flag has improved validation, to better prevent +you from committing an image with a name such as `-m`. Image names with dashes +in them potentially conflict with command line flags. + +* The API now has Improved status codes for `start` and `stop`. Trying to start +a running container will now return a 304 error. + +* Performance has been improved overall. Starting the daemon is faster than in +previous releases. The daemon’s performance has also been improved when it is +working with large numbers of images and containers. + +* Fixed an issue with white-spaces and multi-lines in Dockerfiles. + +##Version 1.1.0 + +###New Features + +*`.dockerignore` support* + +You can now add a `.dockerignore` file next to your `Dockerfile` and Docker will +ignore files and directories specified in that file when sending the build +context to the daemon. Example: +https://github.com/dotcloud/docker/blob/master/.dockerignore + +*Pause containers during commit* + +Doing a commit on a running container was not recommended because you could end +up with files in an inconsistent state (for example, if they were being written +during the commit). Containers are now paused when a commit is made to them. You +can disable this feature by doing a `docker commit --pause=false ` + +*Tailing logs* + +You can now tail the logs of a container. For example, you can get the last ten +lines of a log by using `docker logs --tail 10 `. You can also +follow the logs of a container without having to read the whole log file with +`docker logs --tail 0 -f `. + +*Allow a tar file as context for docker build* + +You can now pass a tar archive to `docker build` as context. This can be used to +automate docker builds, for example: `cat context.tar | docker build -` or +`docker run builder_image | docker build -` + +*Bind mounting your whole filesystem in a container* + +`/` is now allowed as source of `--volumes`. This means you can bind-mount your +whole system in a container if you need to. For example: `docker run -v +/:/my_host ubuntu:ro ls /my_host`. However, it is now forbidden to mount to /. + + +###Other Improvements & Changes + +* Port allocation has been improved. In the previous release, Docker could +prevent you from starting a container with previously allocated ports which +seemed to be in use when in fact they were not. This has been fixed. + +* A bug in `docker save` was introduced in the last release. The `docker save` +command could produce images with invalid metadata. The command now produces +images with correct metadata. + +* Running `docker inspect` in a container now returns which containers it is +linked to. + +* Parsing of the `docker commit` flag has improved validation, to better prevent +you from committing an image with a name such as `-m`. Image names with dashes +in them potentially conflict with command line flags. + +* The API now has Improved status codes for `start` and `stop`. Trying to start +a running container will now return a 304 error. + +* Performance has been improved overall. Starting the daemon is faster than in +previous releases. The daemon’s performance has also been improved when it is +working with large numbers of images and containers. + +* Fixed an issue with white-spaces and multi-lines in Dockerfiles. + +##Version 1.0.0 + +First production-ready release. Prior development history can be found by +searching in [GitHub](https://github.com/docker/docker). From 7f8cdeb18ba14502845efa7e7e90e913d2df23d2 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Fri, 24 Oct 2014 00:23:25 +0000 Subject: [PATCH 14/67] builder: some small fixups + fix a bug where empty entrypoints would not override inheritance. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- builder/dispatchers.go | 10 +++--- builder/evaluator.go | 2 +- builder/parser/parser.go | 5 +-- integration-cli/docker_cli_build_test.go | 43 ++++++++++++++++++++++++ runconfig/merge.go | 5 ++- 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/builder/dispatchers.go b/builder/dispatchers.go index 0c2a58087..e585c4021 100644 --- a/builder/dispatchers.go +++ b/builder/dispatchers.go @@ -195,7 +195,7 @@ func run(b *Builder, args []string, attributes map[string]bool, original string) defer func(cmd []string) { b.Config.Cmd = cmd }(cmd) - log.Debugf("Command to be executed: %v", b.Config.Cmd) + log.Debugf("[BUILDER] Command to be executed: %v", b.Config.Cmd) hit, err := b.probeCache() if err != nil { @@ -261,14 +261,14 @@ func entrypoint(b *Builder, args []string, attributes map[string]bool, original parsed := handleJsonArgs(args, attributes) switch { - case len(parsed) == 0: - // ENTYRPOINT [] - b.Config.Entrypoint = nil case attributes["json"]: // ENTRYPOINT ["echo", "hi"] b.Config.Entrypoint = parsed + case len(parsed) == 0: + // ENTRYPOINT [] + b.Config.Entrypoint = nil default: - // ENTYRPOINT echo hi + // ENTRYPOINT echo hi b.Config.Entrypoint = []string{"/bin/sh", "-c", parsed[0]} } diff --git a/builder/evaluator.go b/builder/evaluator.go index 412261635..aed8d2933 100644 --- a/builder/evaluator.go +++ b/builder/evaluator.go @@ -149,7 +149,7 @@ func (b *Builder) Run(context io.Reader) (string, error) { b.dockerfile = ast // some initializations that would not have been supplied by the caller. - b.Config = &runconfig.Config{Entrypoint: []string{}, Cmd: nil} + b.Config = &runconfig.Config{} b.TmpContainers = map[string]struct{}{} for i, n := range b.dockerfile.Children { diff --git a/builder/parser/parser.go b/builder/parser/parser.go index 5e8bcb5a9..6b0ab7ab8 100644 --- a/builder/parser/parser.go +++ b/builder/parser/parser.go @@ -87,10 +87,11 @@ func parseLine(line string) (string, *Node, error) { if sexp.Value != "" || sexp.Next != nil || sexp.Children != nil { node.Next = sexp - node.Attributes = attrs - node.Original = line } + node.Attributes = attrs + node.Original = line + return "", node, nil } diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 808f64c7e..8be0e01a2 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -1362,6 +1362,49 @@ func TestBuildExpose(t *testing.T) { logDone("build - expose") } +func TestBuildEmptyEntrypointInheritance(t *testing.T) { + name := "testbuildentrypointinheritance" + name2 := "testbuildentrypointinheritance2" + defer deleteImages(name, name2) + + _, err := buildImage(name, + `FROM busybox + ENTRYPOINT ["/bin/echo"]`, + true) + if err != nil { + t.Fatal(err) + } + res, err := inspectField(name, "Config.Entrypoint") + if err != nil { + t.Fatal(err) + } + + expected := "[/bin/echo]" + if res != expected { + t.Fatalf("Entrypoint %s, expected %s", res, expected) + } + + _, err = buildImage(name2, + fmt.Sprintf(`FROM %s + ENTRYPOINT []`, name), + true) + if err != nil { + t.Fatal(err) + } + res, err = inspectField(name2, "Config.Entrypoint") + if err != nil { + t.Fatal(err) + } + + expected = "[]" + + if res != expected { + t.Fatalf("Entrypoint %s, expected %s", res, expected) + } + + logDone("build - empty entrypoint inheritance") +} + func TestBuildEmptyEntrypoint(t *testing.T) { name := "testbuildentrypoint" defer deleteImages(name) diff --git a/runconfig/merge.go b/runconfig/merge.go index 0c60d1df0..64950bf62 100644 --- a/runconfig/merge.go +++ b/runconfig/merge.go @@ -88,7 +88,10 @@ func Merge(userConf, imageConf *Config) error { if len(userConf.Cmd) == 0 { userConf.Cmd = imageConf.Cmd } - userConf.Entrypoint = imageConf.Entrypoint + + if userConf.Entrypoint == nil { + userConf.Entrypoint = imageConf.Entrypoint + } } if userConf.WorkingDir == "" { userConf.WorkingDir = imageConf.WorkingDir From 2dac82eb82b469f94ef26a153d2679663b048ad3 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Sat, 25 Oct 2014 17:58:57 +0000 Subject: [PATCH 15/67] builder: handle escapes without swallowing all of them. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- builder/support.go | 3 +- integration-cli/docker_cli_build_test.go | 86 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/builder/support.go b/builder/support.go index 6c7ac4096..6833457f3 100644 --- a/builder/support.go +++ b/builder/support.go @@ -24,8 +24,9 @@ func (b *Builder) replaceEnv(str string) string { continue } + prefix := match[:idx] stripped := match[idx+2:] - str = strings.Replace(str, match, "$"+stripped, -1) + str = strings.Replace(str, match, prefix+"$"+stripped, -1) continue } diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 8be0e01a2..a4922446c 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -15,6 +15,92 @@ import ( "github.com/docker/docker/pkg/archive" ) +func TestBuildHandleEscapes(t *testing.T) { + name := "testbuildhandleescapes" + + defer deleteImages(name) + + _, err := buildImage(name, + ` + FROM scratch + ENV FOO bar + VOLUME ${FOO} + `, true) + + if err != nil { + t.Fatal(err) + } + + var result map[string]map[string]struct{} + + res, err := inspectFieldJSON(name, "Config.Volumes") + if err != nil { + t.Fatal(err) + } + + if err = unmarshalJSON([]byte(res), &result); err != nil { + t.Fatal(err) + } + + if _, ok := result["bar"]; !ok { + t.Fatal("Could not find volume bar set from env foo in volumes table") + } + + _, err = buildImage(name, + ` + FROM scratch + ENV FOO bar + VOLUME \${FOO} + `, true) + + if err != nil { + t.Fatal(err) + } + + res, err = inspectFieldJSON(name, "Config.Volumes") + if err != nil { + t.Fatal(err) + } + + if err = unmarshalJSON([]byte(res), &result); err != nil { + t.Fatal(err) + } + + if _, ok := result["${FOO}"]; !ok { + t.Fatal("Could not find volume ${FOO} set from env foo in volumes table") + } + + // this test in particular provides *7* backslashes and expects 6 to come back. + // Like above, the first escape is swallowed and the rest are treated as + // literals, this one is just less obvious because of all the character noise. + + _, err = buildImage(name, + ` + FROM scratch + ENV FOO bar + VOLUME \\\\\\\${FOO} + `, true) + + if err != nil { + t.Fatal(err) + } + + res, err = inspectFieldJSON(name, "Config.Volumes") + if err != nil { + t.Fatal(err) + } + + if err = unmarshalJSON([]byte(res), &result); err != nil { + t.Fatal(err) + } + + if _, ok := result[`\\\\\\${FOO}`]; !ok { + t.Fatal(`Could not find volume \\\\\\${FOO} set from env foo in volumes table`) + } + + logDone("build - handle escapes") +} + func TestBuildOnBuildLowercase(t *testing.T) { name := "testbuildonbuildlowercase" name2 := "testbuildonbuildlowercase2" From 463297ffe9d0b671d3b26e13905855ab3ff85d57 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Sat, 25 Oct 2014 18:29:18 +0000 Subject: [PATCH 16/67] builder: whitelist verbs useful for environment replacement. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- builder/evaluator.go | 20 ++- integration-cli/docker_cli_build_test.go | 184 ++++++++++++++++++++++- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/builder/evaluator.go b/builder/evaluator.go index aed8d2933..7884d36ac 100644 --- a/builder/evaluator.go +++ b/builder/evaluator.go @@ -41,6 +41,17 @@ var ( ErrDockerfileEmpty = errors.New("Dockerfile cannot be empty") ) +// Environment variable interpolation will happen on these statements only. +var replaceEnvAllowed = map[string]struct{}{ + "env": {}, + "add": {}, + "copy": {}, + "workdir": {}, + "expose": {}, + "volume": {}, + "user": {}, +} + var evaluateTable map[string]func(*Builder, []string, map[string]bool, string) error func init() { @@ -196,13 +207,18 @@ func (b *Builder) dispatch(stepN int, ast *parser.Node) error { if cmd == "onbuild" { ast = ast.Next.Children[0] - strs = append(strs, b.replaceEnv(ast.Value)) + strs = append(strs, ast.Value) msg += " " + ast.Value } for ast.Next != nil { ast = ast.Next - strs = append(strs, b.replaceEnv(ast.Value)) + var str string + str = ast.Value + if _, ok := replaceEnvAllowed[cmd]; ok { + str = b.replaceEnv(ast.Value) + } + strs = append(strs, str) msg += " " + ast.Value } diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index a4922446c..276a16f29 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -2,6 +2,7 @@ package main import ( "archive/tar" + "encoding/json" "fmt" "io/ioutil" "os" @@ -15,6 +16,186 @@ import ( "github.com/docker/docker/pkg/archive" ) +func TestBuildEnvironmentReplacementUser(t *testing.T) { + name := "testbuildenvironmentreplacement" + defer deleteImages(name) + + _, err := buildImage(name, ` + FROM scratch + ENV user foo + USER ${user} + `, true) + if err != nil { + t.Fatal(err) + } + + res, err := inspectFieldJSON(name, "Config.User") + if err != nil { + t.Fatal(err) + } + + if res != `"foo"` { + t.Fatal("User foo from environment not in Config.User on image") + } + + logDone("build - user environment replacement") +} + +func TestBuildEnvironmentReplacementVolume(t *testing.T) { + name := "testbuildenvironmentreplacement" + defer deleteImages(name) + + _, err := buildImage(name, ` + FROM scratch + ENV volume /quux + VOLUME ${volume} + `, true) + if err != nil { + t.Fatal(err) + } + + res, err := inspectFieldJSON(name, "Config.Volumes") + if err != nil { + t.Fatal(err) + } + + var volumes map[string]interface{} + + if err := json.Unmarshal([]byte(res), &volumes); err != nil { + t.Fatal(err) + } + + if _, ok := volumes["/quux"]; !ok { + t.Fatal("Volume /quux from environment not in Config.Volumes on image") + } + + logDone("build - volume environment replacement") +} + +func TestBuildEnvironmentReplacementExpose(t *testing.T) { + name := "testbuildenvironmentreplacement" + defer deleteImages(name) + + _, err := buildImage(name, ` + FROM scratch + ENV port 80 + EXPOSE ${port} + `, true) + if err != nil { + t.Fatal(err) + } + + res, err := inspectFieldJSON(name, "Config.ExposedPorts") + if err != nil { + t.Fatal(err) + } + + var exposedPorts map[string]interface{} + + if err := json.Unmarshal([]byte(res), &exposedPorts); err != nil { + t.Fatal(err) + } + + if _, ok := exposedPorts["80/tcp"]; !ok { + t.Fatal("Exposed port 80 from environment not in Config.ExposedPorts on image") + } + + logDone("build - expose environment replacement") +} + +func TestBuildEnvironmentReplacementWorkdir(t *testing.T) { + name := "testbuildenvironmentreplacement" + defer deleteImages(name) + + _, err := buildImage(name, ` + FROM busybox + ENV MYWORKDIR /work + RUN mkdir ${MYWORKDIR} + WORKDIR ${MYWORKDIR} + `, true) + + if err != nil { + t.Fatal(err) + } + + logDone("build - workdir environment replacement") +} + +func TestBuildEnvironmentReplacementAddCopy(t *testing.T) { + name := "testbuildenvironmentreplacement" + defer deleteImages(name) + + ctx, err := fakeContext(` + FROM scratch + ENV baz foo + ENV quux bar + ENV dot . + + ADD ${baz} ${dot} + COPY ${quux} ${dot} + `, + map[string]string{ + "foo": "test1", + "bar": "test2", + }) + + if err != nil { + t.Fatal(err) + } + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + t.Fatal(err) + } + + logDone("build - add/copy environment replacement") +} + +func TestBuildEnvironmentReplacementEnv(t *testing.T) { + name := "testbuildenvironmentreplacement" + + defer deleteImages(name) + + _, err := buildImage(name, + ` + FROM scratch + ENV foo foo + ENV bar ${foo} + `, true) + + if err != nil { + t.Fatal(err) + } + + res, err := inspectFieldJSON(name, "Config.Env") + if err != nil { + t.Fatal(err) + } + + envResult := []string{} + + if err = unmarshalJSON([]byte(res), &envResult); err != nil { + t.Fatal(err) + } + + found := false + + for _, env := range envResult { + parts := strings.SplitN(env, "=", 2) + if parts[0] == "bar" { + found = true + if parts[1] != "foo" { + t.Fatal("Could not find replaced var for env `bar`: got %q instead of `foo`", parts[1]) + } + } + } + + if !found { + t.Fatal("Never found the `bar` env variable") + } + + logDone("build - env environment replacement") +} + func TestBuildHandleEscapes(t *testing.T) { name := "testbuildhandleescapes" @@ -170,7 +351,7 @@ func TestBuildEnvOverwrite(t *testing.T) { ` FROM busybox ENV TEST foo - CMD echo \${TEST} + CMD echo ${TEST} `, true) @@ -2547,6 +2728,7 @@ func TestBuildEnvUsage(t *testing.T) { name := "testbuildenvusage" defer deleteImages(name) dockerfile := `FROM busybox +ENV HOME /root ENV PATH $HOME/bin:$PATH ENV PATH /tmp:$PATH RUN [ "$PATH" = "/tmp:$HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ] From 9fc8b7f4e1f88d8573118b7300b18379072418d1 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Mon, 27 Oct 2014 21:15:28 +0000 Subject: [PATCH 17/67] builder: Restore /bin/sh handling in CMD when entrypoint is specified with JSON Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- builder/dispatchers.go | 2 +- integration-cli/docker_cli_build_test.go | 34 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/builder/dispatchers.go b/builder/dispatchers.go index e585c4021..2184e48a8 100644 --- a/builder/dispatchers.go +++ b/builder/dispatchers.go @@ -234,7 +234,7 @@ func run(b *Builder, args []string, attributes map[string]bool, original string) func cmd(b *Builder, args []string, attributes map[string]bool, original string) error { b.Config.Cmd = handleJsonArgs(args, attributes) - if !attributes["json"] && len(b.Config.Entrypoint) == 0 { + if !attributes["json"] { b.Config.Cmd = append([]string{"/bin/sh", "-c"}, b.Config.Cmd...) } diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 276a16f29..0885f9131 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -16,6 +16,40 @@ import ( "github.com/docker/docker/pkg/archive" ) +func TestBuildShCmdJSONEntrypoint(t *testing.T) { + name := "testbuildshcmdjsonentrypoint" + defer deleteImages(name) + + _, err := buildImage( + name, + ` + FROM busybox + ENTRYPOINT ["/bin/echo"] + CMD echo test + `, + true) + + if err != nil { + t.Fatal(err) + } + + out, _, err := runCommandWithOutput( + exec.Command( + dockerBinary, + "run", + name)) + + if err != nil { + t.Fatal(err) + } + + if strings.TrimSpace(out) != "/bin/sh -c echo test" { + t.Fatal("CMD did not contain /bin/sh -c") + } + + logDone("build - CMD should always contain /bin/sh -c when specified without JSON") +} + func TestBuildEnvironmentReplacementUser(t *testing.T) { name := "testbuildenvironmentreplacement" defer deleteImages(name) From e6efbd659606386db4d0b83b98f9e189cf42595c Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 28 Oct 2014 21:20:30 -0400 Subject: [PATCH 18/67] Fix login command Signed-off-by: Tibor Vass --- registry/service.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/registry/service.go b/registry/service.go index 32274f407..7051d9343 100644 --- a/registry/service.go +++ b/registry/service.go @@ -50,9 +50,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { authConfig.ServerAddress = endpoint.String() } - if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { + status, err := Login(authConfig, HTTPRequestFactory(nil)) + if err != nil { return job.Error(err) } + job.Printf("%s\n", status) return engine.StatusOK } From 4e9bbfa90054cd730e81b53b2de67a74306afc95 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 20 Oct 2014 19:17:32 -0400 Subject: [PATCH 19/67] Bump to version v1.3.1 Signed-off-by: Tibor Vass --- CHANGELOG.md | 18 ++++++++++++++++++ VERSION | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5218eea..f958bbd48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 1.3.1 (2014-10-28) + +#### Security +* Prevent fallback to SSL protocols < TLS 1.0 for client, daemon and registry ++ Secure HTTPS connection to registries with certificate verification and without HTTP fallback unless `--insecure-registry` is specified + +#### Runtime +- Fix issue where volumes would not be shared + +#### Client +- Fix issue with `--iptables=false` not automatically setting `--ip-masq=false` +- Fix docker run output to non-TTY stdout + +#### Builder +- Fix escaping `$` for environment variables +- Fix issue with lowercase `onbuild` Dockerfile instruction +- Restrict envrionment variable expansion to `ENV`, `ADD`, `COPY`, `WORKDIR`, `EXPOSE`, `VOLUME` and `USER` + ## 1.3.0 (2014-10-14) #### Notable features since 1.2.0 diff --git a/VERSION b/VERSION index f0bb29e76..3a3cd8cc8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1 From 09c38a8d43cb7fcb128861d0a7f32acaee92115c Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 5 Nov 2014 10:12:57 +0200 Subject: [PATCH 20/67] bump fpm to 1.3.2 Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 34b0460c0..d9c58a921 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,7 +78,7 @@ RUN cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do RUN go get code.google.com/p/go.tools/cmd/cover # TODO replace FPM with some very minimal debhelper stuff -RUN gem install --no-rdoc --no-ri fpm --version 1.0.2 +RUN gem install --no-rdoc --no-ri fpm --version 1.3.2 # Install man page generator RUN mkdir -p /go/src/github.com/cpuguy83 \ From fd9c2ae27dc08114ae1170503217da8d2cb778d6 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Mon, 3 Nov 2014 10:50:16 -0800 Subject: [PATCH 21/67] Fix deadlock in ps exited filter Fixes #8909 Signed-off-by: Alexandr Morozov Conflicts: integration-cli/docker_cli_ps_test.go fixed merge issue caused by missing tests --- daemon/list.go | 2 +- integration-cli/docker_cli_ps_test.go | 78 +++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/daemon/list.go b/daemon/list.go index 35effa434..25360b679 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -93,7 +93,7 @@ func (daemon *Daemon) Containers(job *engine.Job) engine.Status { if len(filt_exited) > 0 && !container.Running { should_skip := true for _, code := range filt_exited { - if code == container.GetExitCode() { + if code == container.ExitCode { should_skip = false break } diff --git a/integration-cli/docker_cli_ps_test.go b/integration-cli/docker_cli_ps_test.go index 4bcf95bf7..f2a7b2ab4 100644 --- a/integration-cli/docker_cli_ps_test.go +++ b/integration-cli/docker_cli_ps_test.go @@ -282,3 +282,81 @@ func TestPsListContainersFilterStatus(t *testing.T) { logDone("ps - test ps filter status") } + +func TestPsListContainersFilterExited(t *testing.T) { + deleteAllContainers() + defer deleteAllContainers() + runCmd := exec.Command(dockerBinary, "run", "--name", "zero1", "busybox", "true") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + t.Fatal(out, err) + } + firstZero, err := getIDByName("zero1") + if err != nil { + t.Fatal(err) + } + + runCmd = exec.Command(dockerBinary, "run", "--name", "zero2", "busybox", "true") + out, _, err = runCommandWithOutput(runCmd) + if err != nil { + t.Fatal(out, err) + } + secondZero, err := getIDByName("zero2") + if err != nil { + t.Fatal(err) + } + + runCmd = exec.Command(dockerBinary, "run", "--name", "nonzero1", "busybox", "false") + out, _, err = runCommandWithOutput(runCmd) + if err == nil { + t.Fatal("Should fail.", out, err) + } + firstNonZero, err := getIDByName("nonzero1") + if err != nil { + t.Fatal(err) + } + + runCmd = exec.Command(dockerBinary, "run", "--name", "nonzero2", "busybox", "false") + out, _, err = runCommandWithOutput(runCmd) + if err == nil { + t.Fatal("Should fail.", out, err) + } + secondNonZero, err := getIDByName("nonzero2") + if err != nil { + t.Fatal(err) + } + + // filter containers by exited=0 + runCmd = exec.Command(dockerBinary, "ps", "-a", "-q", "--no-trunc", "--filter=exited=0") + out, _, err = runCommandWithOutput(runCmd) + if err != nil { + t.Fatal(out, err) + } + ids := strings.Split(strings.TrimSpace(out), "\n") + if len(ids) != 2 { + t.Fatalf("Should be 2 zero exited containerst got %d", len(ids)) + } + if ids[0] != secondZero { + t.Fatalf("First in list should be %q, got %q", secondZero, ids[0]) + } + if ids[1] != firstZero { + t.Fatalf("Second in list should be %q, got %q", firstZero, ids[1]) + } + + runCmd = exec.Command(dockerBinary, "ps", "-a", "-q", "--no-trunc", "--filter=exited=1") + out, _, err = runCommandWithOutput(runCmd) + if err != nil { + t.Fatal(out, err) + } + ids = strings.Split(strings.TrimSpace(out), "\n") + if len(ids) != 2 { + t.Fatalf("Should be 2 zero exited containerst got %d", len(ids)) + } + if ids[0] != secondNonZero { + t.Fatalf("First in list should be %q, got %q", secondNonZero, ids[0]) + } + if ids[1] != firstNonZero { + t.Fatalf("Second in list should be %q, got %q", firstNonZero, ids[1]) + } + logDone("ps - test ps filter exited") +} From 1d1b813d255512002b382b2d2db48984d9b03957 Mon Sep 17 00:00:00 2001 From: shuai-z Date: Sun, 26 Oct 2014 13:55:29 +0800 Subject: [PATCH 22/67] removed redundant Clean The doc (or src) says: The result is Cleaned. http://golang.org/pkg/path/filepath/#Join Signed-off-by: shuai-z --- pkg/symlink/fs.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/symlink/fs.go b/pkg/symlink/fs.go index da9c59067..d76173257 100644 --- a/pkg/symlink/fs.go +++ b/pkg/symlink/fs.go @@ -35,7 +35,6 @@ func FollowSymlinkInScope(link, root string) (string, error) { for _, p := range strings.Split(link, "/") { prev = filepath.Join(prev, p) - prev = filepath.Clean(prev) loopCounter := 0 for { @@ -72,7 +71,7 @@ func FollowSymlinkInScope(link, root string) (string, error) { } else { prev, _ = filepath.Abs(prev) - if prev = filepath.Clean(filepath.Join(filepath.Dir(prev), dest)); len(prev) < len(root) { + if prev = filepath.Join(filepath.Dir(prev), dest); len(prev) < len(root) { prev = filepath.Join(root, filepath.Base(dest)) } } From 662ca4114de579e53e670296615ceb49b8c32d75 Mon Sep 17 00:00:00 2001 From: unclejack Date: Tue, 28 Oct 2014 23:18:45 +0200 Subject: [PATCH 23/67] pkg/symlink: avoid following out of scope Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- pkg/symlink/fs.go | 47 +++++++---- pkg/symlink/fs_test.go | 151 +++++++++++++++++++++++++++++++++--- pkg/symlink/testdata/fs/j/k | 1 + 3 files changed, 172 insertions(+), 27 deletions(-) create mode 120000 pkg/symlink/testdata/fs/j/k diff --git a/pkg/symlink/fs.go b/pkg/symlink/fs.go index d76173257..6ce99c6bd 100644 --- a/pkg/symlink/fs.go +++ b/pkg/symlink/fs.go @@ -12,6 +12,12 @@ const maxLoopCounter = 100 // FollowSymlink will follow an existing link and scope it to the root // path provided. +// The role of this function is to return an absolute path in the root +// or normalize to the root if the symlink leads to a path which is +// outside of the root. +// Errors encountered while attempting to follow the symlink in path +// will be reported. +// Normalizations to the root don't constitute errors. func FollowSymlinkInScope(link, root string) (string, error) { root, err := filepath.Abs(root) if err != nil { @@ -60,25 +66,36 @@ func FollowSymlinkInScope(link, root string) (string, error) { } return "", err } - if stat.Mode()&os.ModeSymlink == os.ModeSymlink { - dest, err := os.Readlink(prev) - if err != nil { - return "", err - } - if path.IsAbs(dest) { - prev = filepath.Join(root, dest) - } else { - prev, _ = filepath.Abs(prev) - - if prev = filepath.Join(filepath.Dir(prev), dest); len(prev) < len(root) { - prev = filepath.Join(root, filepath.Base(dest)) - } - } - } else { + // let's break if we're not dealing with a symlink + if stat.Mode()&os.ModeSymlink != os.ModeSymlink { break } + + // process the symlink + dest, err := os.Readlink(prev) + if err != nil { + return "", err + } + + if path.IsAbs(dest) { + prev = filepath.Join(root, dest) + } else { + prev, _ = filepath.Abs(prev) + + dir := filepath.Dir(prev) + prev = filepath.Join(dir, dest) + if dir == root && !strings.HasPrefix(prev, root) { + prev = root + } + if len(prev) < len(root) || (len(prev) == len(root) && prev != root) { + prev = filepath.Join(root, filepath.Base(dest)) + } + } } } + if prev == "/" { + prev = root + } return prev, nil } diff --git a/pkg/symlink/fs_test.go b/pkg/symlink/fs_test.go index d85fd6da7..0e2f948b6 100644 --- a/pkg/symlink/fs_test.go +++ b/pkg/symlink/fs_test.go @@ -46,6 +46,7 @@ func TestFollowSymLinkUnderLinkedDir(t *testing.T) { if err != nil { t.Fatal(err) } + defer os.RemoveAll(dir) os.Mkdir(filepath.Join(dir, "realdir"), 0700) os.Symlink("realdir", filepath.Join(dir, "linkdir")) @@ -97,25 +98,151 @@ func TestFollowSymLinkRelativeLink(t *testing.T) { } func TestFollowSymLinkRelativeLinkScope(t *testing.T) { - link := "testdata/fs/a/f" + // avoid letting symlink f lead us out of the "testdata" scope + // we don't normalize because symlink f is in scope and there is no + // information leak + { + link := "testdata/fs/a/f" - rewrite, err := FollowSymlinkInScope(link, "testdata") - if err != nil { - t.Fatal(err) + rewrite, err := FollowSymlinkInScope(link, "testdata") + if err != nil { + t.Fatal(err) + } + + if expected := abs(t, "testdata/test"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } } - if expected := abs(t, "testdata/test"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) + // avoid letting symlink f lead us out of the "testdata/fs" scope + // we don't normalize because symlink f is in scope and there is no + // information leak + { + link := "testdata/fs/a/f" + + rewrite, err := FollowSymlinkInScope(link, "testdata/fs") + if err != nil { + t.Fatal(err) + } + + if expected := abs(t, "testdata/fs/test"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } } - link = "testdata/fs/b/h" + // avoid letting symlink g (pointed at by symlink h) take out of scope + // TODO: we should probably normalize to scope here because ../[....]/root + // is out of scope and we leak information + { + link := "testdata/fs/b/h" - rewrite, err = FollowSymlinkInScope(link, "testdata") - if err != nil { - t.Fatal(err) + rewrite, err := FollowSymlinkInScope(link, "testdata") + if err != nil { + t.Fatal(err) + } + + if expected := abs(t, "testdata/root"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } } - if expected := abs(t, "testdata/root"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) + // avoid letting allowing symlink e lead us to ../b + // normalize to the "testdata/fs/a" + { + link := "testdata/fs/a/e" + + rewrite, err := FollowSymlinkInScope(link, "testdata/fs/a") + if err != nil { + t.Fatal(err) + } + + if expected := abs(t, "testdata/fs/a"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } + } + + // avoid letting symlink -> ../directory/file escape from scope + // normalize to "testdata/fs/j" + { + link := "testdata/fs/j/k" + + rewrite, err := FollowSymlinkInScope(link, "testdata/fs/j") + if err != nil { + t.Fatal(err) + } + + if expected := abs(t, "testdata/fs/j"); expected != rewrite { + t.Fatalf("Expected %s got %s", expected, rewrite) + } + } + + // make sure we don't allow escaping to / + // normalize to dir + { + dir, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + linkFile := filepath.Join(dir, "foo") + os.Mkdir(filepath.Join(dir, ""), 0700) + os.Symlink("/", linkFile) + + rewrite, err := FollowSymlinkInScope(linkFile, dir) + if err != nil { + t.Fatal(err) + } + + if rewrite != dir { + t.Fatalf("Expected %s got %s", dir, rewrite) + } + } + + // make sure we don't allow escaping to / + // normalize to dir + { + dir, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + linkFile := filepath.Join(dir, "foo") + os.Mkdir(filepath.Join(dir, ""), 0700) + os.Symlink("/../../", linkFile) + + rewrite, err := FollowSymlinkInScope(linkFile, dir) + if err != nil { + t.Fatal(err) + } + + if rewrite != dir { + t.Fatalf("Expected %s got %s", dir, rewrite) + } + } + + // make sure we stay in scope without leaking information + // this also checks for escaping to / + // normalize to dir + { + dir, err := ioutil.TempDir("", "docker-fs-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + linkFile := filepath.Join(dir, "foo") + os.Mkdir(filepath.Join(dir, ""), 0700) + os.Symlink("../../", linkFile) + + rewrite, err := FollowSymlinkInScope(linkFile, dir) + if err != nil { + t.Fatal(err) + } + + if rewrite != dir { + t.Fatalf("Expected %s got %s", dir, rewrite) + } } } diff --git a/pkg/symlink/testdata/fs/j/k b/pkg/symlink/testdata/fs/j/k new file mode 120000 index 000000000..f559e8fda --- /dev/null +++ b/pkg/symlink/testdata/fs/j/k @@ -0,0 +1 @@ +../i/a \ No newline at end of file From c9379eb3fbbc484c056f5a5e49d8d0b755a29c45 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 3 Nov 2014 22:57:18 +0000 Subject: [PATCH 24/67] Move security opts to HostConfig These settings need to be in the HostConfig so that they are not committed to an image and cannot introduce a security issue. We can safely move this field from the Config to the HostConfig without any regressions because these settings are consumed at container created and used to populate fields on the Container struct. Because of this, existing settings will be honored for containers already created on a daemon with custom security settings and prevent values being consumed via an Image. Signed-off-by: Michael Crosby --- daemon/daemon.go | 11 +++++------ daemon/daemon_unit_test.go | 2 +- daemon/start.go | 3 +++ runconfig/config.go | 2 -- runconfig/hostconfig.go | 2 ++ runconfig/parse.go | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index d069797ac..9f90643c4 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -528,10 +528,10 @@ func (daemon *Daemon) getEntrypointAndArgs(configEntrypoint, configCmd []string) return entrypoint, args } -func parseSecurityOpt(container *Container, config *runconfig.Config) error { +func parseSecurityOpt(container *Container, config *runconfig.HostConfig) error { var ( - label_opts []string - err error + labelOpts []string + err error ) for _, opt := range config.SecurityOpt { @@ -541,7 +541,7 @@ func parseSecurityOpt(container *Container, config *runconfig.Config) error { } switch con[0] { case "label": - label_opts = append(label_opts, con[1]) + labelOpts = append(labelOpts, con[1]) case "apparmor": container.AppArmorProfile = con[1] default: @@ -549,7 +549,7 @@ func parseSecurityOpt(container *Container, config *runconfig.Config) error { } } - container.ProcessLabel, container.MountLabel, err = label.InitLabels(label_opts) + container.ProcessLabel, container.MountLabel, err = label.InitLabels(labelOpts) return err } @@ -583,7 +583,6 @@ func (daemon *Daemon) newContainer(name string, config *runconfig.Config, img *i execCommands: newExecStore(), } container.root = daemon.containerRoot(container.ID) - err = parseSecurityOpt(container, config) return container, err } diff --git a/daemon/daemon_unit_test.go b/daemon/daemon_unit_test.go index f3b899ec8..fbc3302aa 100644 --- a/daemon/daemon_unit_test.go +++ b/daemon/daemon_unit_test.go @@ -8,7 +8,7 @@ import ( func TestParseSecurityOpt(t *testing.T) { container := &Container{} - config := &runconfig.Config{} + config := &runconfig.HostConfig{} // test apparmor config.SecurityOpt = []string{"apparmor:test_profile"} diff --git a/daemon/start.go b/daemon/start.go index f2c375ddc..f72407e3f 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -44,6 +44,9 @@ func (daemon *Daemon) ContainerStart(job *engine.Job) engine.Status { } func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.HostConfig) error { + if err := parseSecurityOpt(container, hostConfig); err != nil { + return err + } // Validate the HostConfig binds. Make sure that: // the source exists for _, bind := range hostConfig.Binds { diff --git a/runconfig/config.go b/runconfig/config.go index 28e85de64..c00110bf7 100644 --- a/runconfig/config.go +++ b/runconfig/config.go @@ -32,7 +32,6 @@ type Config struct { Entrypoint []string NetworkDisabled bool OnBuild []string - SecurityOpt []string } func ContainerConfigFromJob(job *engine.Job) *Config { @@ -56,7 +55,6 @@ func ContainerConfigFromJob(job *engine.Job) *Config { } job.GetenvJson("ExposedPorts", &config.ExposedPorts) job.GetenvJson("Volumes", &config.Volumes) - config.SecurityOpt = job.GetenvList("SecurityOpt") if PortSpecs := job.GetenvList("PortSpecs"); PortSpecs != nil { config.PortSpecs = PortSpecs } diff --git a/runconfig/hostconfig.go b/runconfig/hostconfig.go index 5c4952203..ae75434d4 100644 --- a/runconfig/hostconfig.go +++ b/runconfig/hostconfig.go @@ -56,6 +56,7 @@ type HostConfig struct { CapAdd []string CapDrop []string RestartPolicy RestartPolicy + SecurityOpt []string } // This is used by the create command when you want to set both the @@ -90,6 +91,7 @@ func ContainerHostConfigFromJob(job *engine.Job) *HostConfig { job.GetenvJson("PortBindings", &hostConfig.PortBindings) job.GetenvJson("Devices", &hostConfig.Devices) job.GetenvJson("RestartPolicy", &hostConfig.RestartPolicy) + hostConfig.SecurityOpt = job.GetenvList("SecurityOpt") if Binds := job.GetenvList("Binds"); Binds != nil { hostConfig.Binds = Binds } diff --git a/runconfig/parse.go b/runconfig/parse.go index 42ec68898..43976f604 100644 --- a/runconfig/parse.go +++ b/runconfig/parse.go @@ -256,7 +256,6 @@ func Parse(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Config, Volumes: flVolumes.GetMap(), Entrypoint: entrypoint, WorkingDir: *flWorkingDir, - SecurityOpt: flSecurityOpt.GetAll(), } hostConfig := &HostConfig{ @@ -276,6 +275,7 @@ func Parse(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Config, CapAdd: flCapAdd.GetAll(), CapDrop: flCapDrop.GetAll(), RestartPolicy: restartPolicy, + SecurityOpt: flSecurityOpt.GetAll(), } if sysInfo != nil && flMemory > 0 && !sysInfo.SwapLimit { From 0573b17b24b66c26e8311d55dff4d81923030f75 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 3 Nov 2014 23:00:49 +0000 Subject: [PATCH 25/67] Add AppArmorProfile to container inspect json Signed-off-by: Michael Crosby --- daemon/inspect.go | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/inspect.go b/daemon/inspect.go index 396ca0227..cf2ed644d 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -47,6 +47,7 @@ func (daemon *Daemon) ContainerInspect(job *engine.Job) engine.Status { out.Set("ProcessLabel", container.ProcessLabel) out.SetJson("Volumes", container.Volumes) out.SetJson("VolumesRW", container.VolumesRW) + out.SetJson("AppArmorProfile", container.AppArmorProfile) if children, err := daemon.Children(container.Name); err == nil { for linkAlias, child := range children { From d51a02091ca5485ed42b336d3df8c6c4ab2d9a6b Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 30 Oct 2014 14:48:30 +0200 Subject: [PATCH 26/67] pkg/reexec: move reexec code to a new package Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Conflicts: integration/runtime_test.go fixed imports --- daemon/execdriver/lxc/init.go | 2 +- daemon/execdriver/native/exec.go | 2 +- daemon/execdriver/native/init.go | 2 +- daemon/networkdriver/portmapper/proxy.go | 2 +- docker/docker.go | 2 +- dockerinit/dockerinit.go | 2 +- integration/runtime_test.go | 2 +- pkg/reexec/MAINTAINERS | 1 + {reexec => pkg/reexec}/README.md | 0 {reexec => pkg/reexec}/reexec.go | 0 10 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 pkg/reexec/MAINTAINERS rename {reexec => pkg/reexec}/README.md (100%) rename {reexec => pkg/reexec}/reexec.go (100%) diff --git a/daemon/execdriver/lxc/init.go b/daemon/execdriver/lxc/init.go index 2a91bbb5f..680f53e1a 100644 --- a/daemon/execdriver/lxc/init.go +++ b/daemon/execdriver/lxc/init.go @@ -13,7 +13,7 @@ import ( "strings" "syscall" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" "github.com/docker/libcontainer/netlink" ) diff --git a/daemon/execdriver/native/exec.go b/daemon/execdriver/native/exec.go index 0f7e5c07b..84ad09672 100644 --- a/daemon/execdriver/native/exec.go +++ b/daemon/execdriver/native/exec.go @@ -11,7 +11,7 @@ import ( "runtime" "github.com/docker/docker/daemon/execdriver" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" "github.com/docker/libcontainer" "github.com/docker/libcontainer/namespaces" ) diff --git a/daemon/execdriver/native/init.go b/daemon/execdriver/native/init.go index 7021eeb67..c1c988d93 100644 --- a/daemon/execdriver/native/init.go +++ b/daemon/execdriver/native/init.go @@ -10,7 +10,7 @@ import ( "path/filepath" "runtime" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" "github.com/docker/libcontainer" "github.com/docker/libcontainer/namespaces" "github.com/docker/libcontainer/syncpipe" diff --git a/daemon/networkdriver/portmapper/proxy.go b/daemon/networkdriver/portmapper/proxy.go index 341f0605e..af20469ed 100644 --- a/daemon/networkdriver/portmapper/proxy.go +++ b/daemon/networkdriver/portmapper/proxy.go @@ -14,7 +14,7 @@ import ( "time" "github.com/docker/docker/pkg/proxy" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" ) const userlandProxyCommandName = "docker-proxy" diff --git a/docker/docker.go b/docker/docker.go index f0cbb6f6a..12900ecb1 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -13,7 +13,7 @@ import ( "github.com/docker/docker/api/client" "github.com/docker/docker/dockerversion" flag "github.com/docker/docker/pkg/mflag" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/utils" ) diff --git a/dockerinit/dockerinit.go b/dockerinit/dockerinit.go index c5bba782b..a6754b05a 100644 --- a/dockerinit/dockerinit.go +++ b/dockerinit/dockerinit.go @@ -3,7 +3,7 @@ package main import ( _ "github.com/docker/docker/daemon/execdriver/lxc" _ "github.com/docker/docker/daemon/execdriver/native" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" ) func main() { diff --git a/integration/runtime_test.go b/integration/runtime_test.go index 6720485e2..0a9a74cf9 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -22,7 +22,7 @@ import ( "github.com/docker/docker/nat" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/log" - "github.com/docker/docker/reexec" + "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" ) diff --git a/pkg/reexec/MAINTAINERS b/pkg/reexec/MAINTAINERS new file mode 100644 index 000000000..e48a0c7d4 --- /dev/null +++ b/pkg/reexec/MAINTAINERS @@ -0,0 +1 @@ +Michael Crosby (@crosbymichael) diff --git a/reexec/README.md b/pkg/reexec/README.md similarity index 100% rename from reexec/README.md rename to pkg/reexec/README.md diff --git a/reexec/reexec.go b/pkg/reexec/reexec.go similarity index 100% rename from reexec/reexec.go rename to pkg/reexec/reexec.go From 3ab5251f5620e7b8b052ee331f9ac1fbdbb725b1 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Mon, 27 Oct 2014 11:38:22 -0700 Subject: [PATCH 27/67] Use archive.CopyWithTar in vfs.Create The vfs storage driver currently shells out to the `cp` binary on the host system to perform an 'archive' copy of the base image to a new directory. The archive option preserves the modified time of the files which are created but there was an issue where it was unable to preserve the modified time of copied symbolic links on some host systems with an outdated version of `cp`. This change no longer relies on the host system implementation and instead utilizes the `CopyWithTar` function found in `pkg/archive` which is used to copy from source to destination directory using a Tar archive, which should correctly preserve file attributes. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- daemon/graphdriver/vfs/driver.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/daemon/graphdriver/vfs/driver.go b/daemon/graphdriver/vfs/driver.go index a186060d0..1076eb38d 100644 --- a/daemon/graphdriver/vfs/driver.go +++ b/daemon/graphdriver/vfs/driver.go @@ -8,6 +8,7 @@ import ( "path" "github.com/docker/docker/daemon/graphdriver" + "github.com/docker/docker/pkg/archive" "github.com/docker/libcontainer/label" ) @@ -46,21 +47,6 @@ func isGNUcoreutils() bool { return false } -func copyDir(src, dst string) error { - argv := make([]string, 0, 4) - - if isGNUcoreutils() { - argv = append(argv, "-aT", "--reflink=auto", src, dst) - } else { - argv = append(argv, "-a", src+"/.", dst+"/.") - } - - if output, err := exec.Command("cp", argv...).CombinedOutput(); err != nil { - return fmt.Errorf("Error VFS copying directory: %s (%s)", err, output) - } - return nil -} - func (d *Driver) Create(id, parent string) error { dir := d.dir(id) if err := os.MkdirAll(path.Dir(dir), 0700); err != nil { @@ -80,7 +66,7 @@ func (d *Driver) Create(id, parent string) error { if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := copyDir(parentDir, dir); err != nil { + if err := archive.CopyWithTar(parentDir, dir); err != nil { return err } return nil From 1d4a82365bfa215767fc4a4cc7e4d5d10dbcd0e4 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 6 Nov 2014 20:01:37 +0200 Subject: [PATCH 28/67] pkg/archive: add interface for Untar Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- pkg/archive/archive.go | 71 +++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 7d9103e10..cdac22840 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -35,10 +35,17 @@ type ( Compression Compression NoLchown bool } + + // Archiver allows the reuse of most utility functions of this package + // with a pluggable Untar function. + Archiver struct { + Untar func(io.Reader, string, *TarOptions) error + } ) var ( ErrNotImplemented = errors.New("Function not implemented") + defaultArchiver = &Archiver{Untar} ) const ( @@ -508,45 +515,47 @@ loop: return nil } -// TarUntar is a convenience function which calls Tar and Untar, with -// the output of one piped into the other. If either Tar or Untar fails, -// TarUntar aborts and returns the error. -func TarUntar(src string, dst string) error { +func (archiver *Archiver) TarUntar(src, dst string) error { log.Debugf("TarUntar(%s %s)", src, dst) archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) if err != nil { return err } defer archive.Close() - return Untar(archive, dst, nil) + return archiver.Untar(archive, dst, nil) } -// UntarPath is a convenience function which looks for an archive -// at filesystem path `src`, and unpacks it at `dst`. -func UntarPath(src, dst string) error { +// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. +// If either Tar or Untar fails, TarUntar aborts and returns the error. +func TarUntar(src, dst string) error { + return defaultArchiver.TarUntar(src, dst) +} + +func (archiver *Archiver) UntarPath(src, dst string) error { archive, err := os.Open(src) if err != nil { return err } defer archive.Close() - if err := Untar(archive, dst, nil); err != nil { + if err := archiver.Untar(archive, dst, nil); err != nil { return err } return nil } -// CopyWithTar creates a tar archive of filesystem path `src`, and -// unpacks it at filesystem path `dst`. -// The archive is streamed directly with fixed buffering and no -// intermediary disk IO. -// -func CopyWithTar(src, dst string) error { +// UntarPath is a convenience function which looks for an archive +// at filesystem path `src`, and unpacks it at `dst`. +func UntarPath(src, dst string) error { + return defaultArchiver.UntarPath(src, dst) +} + +func (archiver *Archiver) CopyWithTar(src, dst string) error { srcSt, err := os.Stat(src) if err != nil { return err } if !srcSt.IsDir() { - return CopyFileWithTar(src, dst) + return archiver.CopyFileWithTar(src, dst) } // Create dst, copy src's content into it log.Debugf("Creating dest directory: %s", dst) @@ -554,16 +563,18 @@ func CopyWithTar(src, dst string) error { return err } log.Debugf("Calling TarUntar(%s, %s)", src, dst) - return TarUntar(src, dst) + return archiver.TarUntar(src, dst) } -// CopyFileWithTar emulates the behavior of the 'cp' command-line -// for a single file. It copies a regular file from path `src` to -// path `dst`, and preserves all its metadata. -// -// If `dst` ends with a trailing slash '/', the final destination path -// will be `dst/base(src)`. -func CopyFileWithTar(src, dst string) (err error) { +// CopyWithTar creates a tar archive of filesystem path `src`, and +// unpacks it at filesystem path `dst`. +// The archive is streamed directly with fixed buffering and no +// intermediary disk IO. +func CopyWithTar(src, dst string) error { + return defaultArchiver.CopyWithTar(src, dst) +} + +func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { log.Debugf("CopyFileWithTar(%s, %s)", src, dst) srcSt, err := os.Stat(src) if err != nil { @@ -611,7 +622,17 @@ func CopyFileWithTar(src, dst string) (err error) { err = er } }() - return Untar(r, filepath.Dir(dst), nil) + return archiver.Untar(r, filepath.Dir(dst), nil) +} + +// CopyFileWithTar emulates the behavior of the 'cp' command-line +// for a single file. It copies a regular file from path `src` to +// path `dst`, and preserves all its metadata. +// +// If `dst` ends with a trailing slash '/', the final destination path +// will be `dst/base(src)`. +func CopyFileWithTar(src, dst string) (err error) { + return defaultArchiver.CopyFileWithTar(src, dst) } // CmdStream executes a command, and returns its stdout as a stream. From 0357b26c1b5ecc3a4d0d1b2e7cf63ea8e5f70eba Mon Sep 17 00:00:00 2001 From: unclejack Date: Wed, 29 Oct 2014 21:06:51 +0200 Subject: [PATCH 29/67] add pkg/chrootarchive and use it on the daemon Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- builder/internals.go | 11 +++-- daemon/graphdriver/aufs/aufs.go | 3 +- daemon/graphdriver/fsdiff.go | 3 +- daemon/graphdriver/vfs/driver.go | 4 +- daemon/volumes.go | 4 +- pkg/chrootarchive/archive.go | 76 +++++++++++++++++++++++++++++++ pkg/chrootarchive/diff.go | 38 ++++++++++++++++ pkg/chrootarchive/init.go | 18 ++++++++ pkg/reexec/command_linux.go | 18 ++++++++ pkg/reexec/command_unsupported.go | 11 +++++ pkg/reexec/reexec.go | 3 -- 11 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 pkg/chrootarchive/archive.go create mode 100644 pkg/chrootarchive/diff.go create mode 100644 pkg/chrootarchive/init.go create mode 100644 pkg/reexec/command_linux.go create mode 100644 pkg/reexec/command_unsupported.go diff --git a/builder/internals.go b/builder/internals.go index 5fd03f774..5d0d197a0 100644 --- a/builder/internals.go +++ b/builder/internals.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/daemon" imagepkg "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/promise" @@ -46,7 +47,9 @@ func (b *Builder) readContext(context io.Reader) error { if b.context, err = tarsum.NewTarSum(decompressedStream, true, tarsum.Version0); err != nil { return err } - if err := archive.Untar(b.context, tmpdirPath, nil); err != nil { + + os.MkdirAll(tmpdirPath, 0700) + if err := chrootarchive.Untar(b.context, tmpdirPath, nil); err != nil { return err } @@ -620,7 +623,7 @@ func (b *Builder) addContext(container *daemon.Container, orig, dest string, dec } // try to successfully untar the orig - if err := archive.UntarPath(origPath, tarDest); err == nil { + if err := chrootarchive.UntarPath(origPath, tarDest); err == nil { return nil } else if err != io.EOF { log.Debugf("Couldn't untar %s to %s: %s", origPath, tarDest, err) @@ -630,7 +633,7 @@ func (b *Builder) addContext(container *daemon.Container, orig, dest string, dec if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } - if err := archive.CopyWithTar(origPath, destPath); err != nil { + if err := chrootarchive.CopyWithTar(origPath, destPath); err != nil { return err } @@ -643,7 +646,7 @@ func (b *Builder) addContext(container *daemon.Container, orig, dest string, dec } func copyAsDirectory(source, destination string, destinationExists bool) error { - if err := archive.CopyWithTar(source, destination); err != nil { + if err := chrootarchive.CopyWithTar(source, destination); err != nil { return err } diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index 8e3ae0b18..bbaad08e3 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -32,6 +32,7 @@ import ( "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/log" mountpk "github.com/docker/docker/pkg/mount" "github.com/docker/docker/utils" @@ -304,7 +305,7 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { } func (a *Driver) applyDiff(id string, diff archive.ArchiveReader) error { - return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil) + return chrootarchive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil) } // DiffSize calculates the changes between the specified id diff --git a/daemon/graphdriver/fsdiff.go b/daemon/graphdriver/fsdiff.go index 5e9d32c1c..14e27d60c 100644 --- a/daemon/graphdriver/fsdiff.go +++ b/daemon/graphdriver/fsdiff.go @@ -5,6 +5,7 @@ import ( "time" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" @@ -120,7 +121,7 @@ func (gdw *naiveDiffDriver) ApplyDiff(id, parent string, diff archive.ArchiveRea start := time.Now().UTC() log.Debugf("Start untar layer") - if err = archive.ApplyLayer(layerFs, diff); err != nil { + if err = chrootarchive.ApplyLayer(layerFs, diff); err != nil { return } log.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds()) diff --git a/daemon/graphdriver/vfs/driver.go b/daemon/graphdriver/vfs/driver.go index 1076eb38d..aa104500b 100644 --- a/daemon/graphdriver/vfs/driver.go +++ b/daemon/graphdriver/vfs/driver.go @@ -8,7 +8,7 @@ import ( "path" "github.com/docker/docker/daemon/graphdriver" - "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/libcontainer/label" ) @@ -66,7 +66,7 @@ func (d *Driver) Create(id, parent string) error { if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := archive.CopyWithTar(parentDir, dir); err != nil { + if err := chrootarchive.CopyWithTar(parentDir, dir); err != nil { return err } return nil diff --git a/daemon/volumes.go b/daemon/volumes.go index b34d9678c..ec0ccd023 100644 --- a/daemon/volumes.go +++ b/daemon/volumes.go @@ -10,7 +10,7 @@ import ( "syscall" "github.com/docker/docker/daemon/execdriver" - "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/volumes" @@ -302,7 +302,7 @@ func copyExistingContents(source, destination string) error { if len(srcList) == 0 { // If the source volume is empty copy files from the root into the volume - if err := archive.CopyWithTar(source, destination); err != nil { + if err := chrootarchive.CopyWithTar(source, destination); err != nil { return err } } diff --git a/pkg/chrootarchive/archive.go b/pkg/chrootarchive/archive.go new file mode 100644 index 000000000..f1df57ca5 --- /dev/null +++ b/pkg/chrootarchive/archive.go @@ -0,0 +1,76 @@ +package chrootarchive + +import ( + "flag" + "fmt" + "io" + "os" + "runtime" + "syscall" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/reexec" +) + +func untar() { + runtime.LockOSThread() + flag.Parse() + + if err := syscall.Chroot(flag.Arg(0)); err != nil { + fatal(err) + } + if err := syscall.Chdir("/"); err != nil { + fatal(err) + } + if err := archive.Untar(os.Stdin, "/", nil); err != nil { + fatal(err) + } + os.Exit(0) +} + +var ( + chrootArchiver = &archive.Archiver{Untar} +) + +func Untar(archive io.Reader, dest string, options *archive.TarOptions) error { + if _, err := os.Stat(dest); os.IsNotExist(err) { + if err := os.MkdirAll(dest, 0777); err != nil { + return err + } + } + cmd := reexec.Command("docker-untar", dest) + cmd.Stdin = archive + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Untar %s %s", err, out) + } + return nil +} + +func TarUntar(src, dst string) error { + return chrootArchiver.TarUntar(src, dst) +} + +// CopyWithTar creates a tar archive of filesystem path `src`, and +// unpacks it at filesystem path `dst`. +// The archive is streamed directly with fixed buffering and no +// intermediary disk IO. +func CopyWithTar(src, dst string) error { + return chrootArchiver.CopyWithTar(src, dst) +} + +// CopyFileWithTar emulates the behavior of the 'cp' command-line +// for a single file. It copies a regular file from path `src` to +// path `dst`, and preserves all its metadata. +// +// If `dst` ends with a trailing slash '/', the final destination path +// will be `dst/base(src)`. +func CopyFileWithTar(src, dst string) (err error) { + return chrootArchiver.CopyFileWithTar(src, dst) +} + +// UntarPath is a convenience function which looks for an archive +// at filesystem path `src`, and unpacks it at `dst`. +func UntarPath(src, dst string) error { + return chrootArchiver.UntarPath(src, dst) +} diff --git a/pkg/chrootarchive/diff.go b/pkg/chrootarchive/diff.go new file mode 100644 index 000000000..2133200c6 --- /dev/null +++ b/pkg/chrootarchive/diff.go @@ -0,0 +1,38 @@ +package chrootarchive + +import ( + "flag" + "fmt" + "os" + "runtime" + "syscall" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/reexec" +) + +func applyLayer() { + runtime.LockOSThread() + flag.Parse() + + if err := syscall.Chroot(flag.Arg(0)); err != nil { + fatal(err) + } + if err := syscall.Chdir("/"); err != nil { + fatal(err) + } + if err := archive.ApplyLayer("/", os.Stdin); err != nil { + fatal(err) + } + os.Exit(0) +} + +func ApplyLayer(dest string, layer archive.ArchiveReader) error { + cmd := reexec.Command("docker-applyLayer", dest) + cmd.Stdin = layer + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("ApplyLayer %s %s", err, out) + } + return nil +} diff --git a/pkg/chrootarchive/init.go b/pkg/chrootarchive/init.go new file mode 100644 index 000000000..b548e9fe7 --- /dev/null +++ b/pkg/chrootarchive/init.go @@ -0,0 +1,18 @@ +package chrootarchive + +import ( + "fmt" + "os" + + "github.com/docker/docker/pkg/reexec" +) + +func init() { + reexec.Register("docker-untar", untar) + reexec.Register("docker-applyLayer", applyLayer) +} + +func fatal(err error) { + fmt.Fprint(os.Stderr, err) + os.Exit(1) +} diff --git a/pkg/reexec/command_linux.go b/pkg/reexec/command_linux.go new file mode 100644 index 000000000..8dc3f3a4a --- /dev/null +++ b/pkg/reexec/command_linux.go @@ -0,0 +1,18 @@ +// +build linux + +package reexec + +import ( + "os/exec" + "syscall" +) + +func Command(args ...string) *exec.Cmd { + return &exec.Cmd{ + Path: Self(), + Args: args, + SysProcAttr: &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, + }, + } +} diff --git a/pkg/reexec/command_unsupported.go b/pkg/reexec/command_unsupported.go new file mode 100644 index 000000000..a579318e8 --- /dev/null +++ b/pkg/reexec/command_unsupported.go @@ -0,0 +1,11 @@ +// +build !linux + +package reexec + +import ( + "os/exec" +) + +func Command(args ...string) *exec.Cmd { + return nil +} diff --git a/pkg/reexec/reexec.go b/pkg/reexec/reexec.go index 136b905bd..774e71c76 100644 --- a/pkg/reexec/reexec.go +++ b/pkg/reexec/reexec.go @@ -27,19 +27,16 @@ func Init() bool { return true } - return false } // Self returns the path to the current processes binary func Self() string { name := os.Args[0] - if filepath.Base(name) == name { if lp, err := exec.LookPath(name); err == nil { name = lp } } - return name } From 3ac6394b8082d4700483d52fbfe54914be537d9e Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Sat, 8 Nov 2014 10:38:42 -0500 Subject: [PATCH 30/67] pkg/chrootarchive: pass TarOptions via CLI arg Signed-off-by: Tibor Vass --- builder/internals.go | 1 - graph/load.go | 3 ++- pkg/chrootarchive/archive.go | 18 ++++++++++++-- pkg/chrootarchive/archive_test.go | 39 +++++++++++++++++++++++++++++++ pkg/chrootarchive/init.go | 1 + 5 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 pkg/chrootarchive/archive_test.go diff --git a/builder/internals.go b/builder/internals.go index 5d0d197a0..fa8b9f703 100644 --- a/builder/internals.go +++ b/builder/internals.go @@ -48,7 +48,6 @@ func (b *Builder) readContext(context io.Reader) error { return err } - os.MkdirAll(tmpdirPath, 0700) if err := chrootarchive.Untar(b.context, tmpdirPath, nil); err != nil { return err } diff --git a/graph/load.go b/graph/load.go index 753f31d2c..fcbeef619 100644 --- a/graph/load.go +++ b/graph/load.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/log" ) @@ -53,7 +54,7 @@ func (s *TagStore) CmdLoad(job *engine.Job) engine.Status { excludes[i] = k i++ } - if err := archive.Untar(repoFile, repoDir, &archive.TarOptions{Excludes: excludes}); err != nil { + if err := chrootarchive.Untar(repoFile, repoDir, &archive.TarOptions{Excludes: excludes}); err != nil { return job.Error(err) } diff --git a/pkg/chrootarchive/archive.go b/pkg/chrootarchive/archive.go index f1df57ca5..fc2bea2c4 100644 --- a/pkg/chrootarchive/archive.go +++ b/pkg/chrootarchive/archive.go @@ -1,11 +1,14 @@ package chrootarchive import ( + "bytes" + "encoding/json" "flag" "fmt" "io" "os" "runtime" + "strings" "syscall" "github.com/docker/docker/pkg/archive" @@ -22,7 +25,12 @@ func untar() { if err := syscall.Chdir("/"); err != nil { fatal(err) } - if err := archive.Untar(os.Stdin, "/", nil); err != nil { + options := new(archive.TarOptions) + dec := json.NewDecoder(strings.NewReader(flag.Arg(1))) + if err := dec.Decode(options); err != nil { + fatal(err) + } + if err := archive.Untar(os.Stdin, "/", options); err != nil { fatal(err) } os.Exit(0) @@ -33,12 +41,18 @@ var ( ) func Untar(archive io.Reader, dest string, options *archive.TarOptions) error { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + if err := enc.Encode(options); err != nil { + return fmt.Errorf("Untar json encode: %v", err) + } if _, err := os.Stat(dest); os.IsNotExist(err) { if err := os.MkdirAll(dest, 0777); err != nil { return err } } - cmd := reexec.Command("docker-untar", dest) + + cmd := reexec.Command("docker-untar", dest, buf.String()) cmd.Stdin = archive out, err := cmd.CombinedOutput() if err != nil { diff --git a/pkg/chrootarchive/archive_test.go b/pkg/chrootarchive/archive_test.go new file mode 100644 index 000000000..aeac44874 --- /dev/null +++ b/pkg/chrootarchive/archive_test.go @@ -0,0 +1,39 @@ +package chrootarchive + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/docker/docker/pkg/archive" +) + +func TestChrootTarUntar(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "docker-TestChrootTarUntar") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + src := filepath.Join(tmpdir, "src") + if err := os.MkdirAll(src, 0700); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(src, "toto"), []byte("hello toto"), 0644); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(src, "lolo"), []byte("hello lolo"), 0644); err != nil { + t.Fatal(err) + } + stream, err := archive.Tar(src, archive.Uncompressed) + if err != nil { + t.Fatal(err) + } + dest := filepath.Join(tmpdir, "src") + if err := os.MkdirAll(dest, 0700); err != nil { + t.Fatal(err) + } + if err := Untar(stream, dest, &archive.TarOptions{Excludes: []string{"lolo"}}); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/chrootarchive/init.go b/pkg/chrootarchive/init.go index b548e9fe7..f05698f65 100644 --- a/pkg/chrootarchive/init.go +++ b/pkg/chrootarchive/init.go @@ -10,6 +10,7 @@ import ( func init() { reexec.Register("docker-untar", untar) reexec.Register("docker-applyLayer", applyLayer) + reexec.Init() } func fatal(err error) { From 8d90b0faf8e289edc2b4c13c6f312995745f161b Mon Sep 17 00:00:00 2001 From: unclejack Date: Tue, 11 Nov 2014 13:02:14 +0200 Subject: [PATCH 31/67] don't call reexec.Init from chrootarchive Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- daemon/graphdriver/aufs/aufs_test.go | 10 ++++++++-- daemon/graphdriver/vfs/vfs_test.go | 9 ++++++++- graph/pools_test.go | 10 +++++++++- pkg/chrootarchive/archive_test.go | 5 +++++ pkg/chrootarchive/init.go | 1 - 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/daemon/graphdriver/aufs/aufs_test.go b/daemon/graphdriver/aufs/aufs_test.go index cc5b3a203..0c940bc9e 100644 --- a/daemon/graphdriver/aufs/aufs_test.go +++ b/daemon/graphdriver/aufs/aufs_test.go @@ -4,18 +4,24 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "github.com/docker/docker/daemon/graphdriver" - "github.com/docker/docker/pkg/archive" "io/ioutil" "os" "path" "testing" + + "github.com/docker/docker/daemon/graphdriver" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/reexec" ) var ( tmp = path.Join(os.TempDir(), "aufs-tests", "aufs") ) +func init() { + reexec.Init() +} + func testInit(dir string, t *testing.T) graphdriver.Driver { d, err := Init(dir, nil) if err != nil { diff --git a/daemon/graphdriver/vfs/vfs_test.go b/daemon/graphdriver/vfs/vfs_test.go index eaf70f59d..1ee6ae4a9 100644 --- a/daemon/graphdriver/vfs/vfs_test.go +++ b/daemon/graphdriver/vfs/vfs_test.go @@ -1,10 +1,17 @@ package vfs import ( - "github.com/docker/docker/daemon/graphdriver/graphtest" "testing" + + "github.com/docker/docker/daemon/graphdriver/graphtest" + + "github.com/docker/docker/pkg/reexec" ) +func init() { + reexec.Init() +} + // This avoids creating a new driver for each test if all tests are run // Make sure to put new tests between TestVfsSetup and TestVfsTeardown func TestVfsSetup(t *testing.T) { diff --git a/graph/pools_test.go b/graph/pools_test.go index 785a4bd12..129a5e1fe 100644 --- a/graph/pools_test.go +++ b/graph/pools_test.go @@ -1,6 +1,14 @@ package graph -import "testing" +import ( + "testing" + + "github.com/docker/docker/pkg/reexec" +) + +func init() { + reexec.Init() +} func TestPools(t *testing.T) { s := &TagStore{ diff --git a/pkg/chrootarchive/archive_test.go b/pkg/chrootarchive/archive_test.go index aeac44874..69e18e319 100644 --- a/pkg/chrootarchive/archive_test.go +++ b/pkg/chrootarchive/archive_test.go @@ -7,8 +7,13 @@ import ( "testing" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/reexec" ) +func init() { + reexec.Init() +} + func TestChrootTarUntar(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootTarUntar") if err != nil { diff --git a/pkg/chrootarchive/init.go b/pkg/chrootarchive/init.go index f05698f65..b548e9fe7 100644 --- a/pkg/chrootarchive/init.go +++ b/pkg/chrootarchive/init.go @@ -10,7 +10,6 @@ import ( func init() { reexec.Register("docker-untar", untar) reexec.Register("docker-applyLayer", applyLayer) - reexec.Init() } func fatal(err error) { From 967f80f3cceb96f85a4795d42eeb7b84ae0ce24a Mon Sep 17 00:00:00 2001 From: Thomas Orozco Date: Thu, 23 Oct 2014 14:50:06 +0200 Subject: [PATCH 32/67] Fix: Failed Start breaks VolumesFrom Running parseVolumesFromSpec on all VolumesFrom specs before initialize any mounts endures that we don't leave container.Volumes in an inconsistent (partially initialized) if one of out mount groups is not available (e.g. the container we're trying to mount from does not exist). Keeping container.Volumes in a consistent state ensures that next time we Start() the container, it'll run prepareVolumes() again. The attached test demonstrates that when a container fails to start due to a missing container specified in VolumesFrom, it "remembers" a Volume that worked. Fixes: #8726 Signed-off-by: Thomas Orozco Conflicts: integration-cli/docker_cli_start_test.go cli integration test --- daemon/volumes.go | 9 ++++++-- integration-cli/docker_cli_start_test.go | 29 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/daemon/volumes.go b/daemon/volumes.go index ec0ccd023..f98baa1c7 100644 --- a/daemon/volumes.go +++ b/daemon/volumes.go @@ -191,15 +191,20 @@ func parseBindMountSpec(spec string) (string, string, bool, error) { func (container *Container) applyVolumesFrom() error { volumesFrom := container.hostConfig.VolumesFrom + mountGroups := make([]map[string]*Mount, 0, len(volumesFrom)) + for _, spec := range volumesFrom { - mounts, err := parseVolumesFromSpec(container.daemon, spec) + mountGroup, err := parseVolumesFromSpec(container.daemon, spec) if err != nil { return err } + mountGroups = append(mountGroups, mountGroup) + } + for _, mounts := range mountGroups { for _, mnt := range mounts { mnt.container = container - if err = mnt.initialize(); err != nil { + if err := mnt.initialize(); err != nil { return err } } diff --git a/integration-cli/docker_cli_start_test.go b/integration-cli/docker_cli_start_test.go index 18ad96aef..addc781ca 100644 --- a/integration-cli/docker_cli_start_test.go +++ b/integration-cli/docker_cli_start_test.go @@ -2,6 +2,7 @@ package main import ( "os/exec" + "strings" "testing" "time" ) @@ -36,3 +37,31 @@ func TestStartAttachReturnsOnError(t *testing.T) { logDone("start - error on start with attach exits") } + +// gh#8726: a failed Start() breaks --volumes-from on subsequent Start()'s +func TestStartVolumesFromFailsCleanly(t *testing.T) { + defer deleteAllContainers() + + // Create the first data volume + cmd(t, "run", "-d", "--name", "data_before", "-v", "/foo", "busybox") + + // Expect this to fail because the data test after contaienr doesn't exist yet + if _, err := runCommand(exec.Command(dockerBinary, "run", "-d", "--name", "consumer", "--volumes-from", "data_before", "--volumes-from", "data_after", "busybox")); err == nil { + t.Fatal("Expected error but got none") + } + + // Create the second data volume + cmd(t, "run", "-d", "--name", "data_after", "-v", "/bar", "busybox") + + // Now, all the volumes should be there + cmd(t, "start", "consumer") + + // Check that we have the volumes we want + out, _, _ := cmd(t, "inspect", "--format='{{ len .Volumes }}'", "consumer") + n_volumes := strings.Trim(out, " \r\n'") + if n_volumes != "2" { + t.Fatalf("Missing volumes: expected 2, got %s", n_volumes) + } + + logDone("start - missing containers in --volumes-from did not affect subsequent runs") +} From ea361c0476d03f44051289d8e3c128d699c76acc Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 20 Oct 2014 15:35:48 -0400 Subject: [PATCH 33/67] archive: add breakout tests Signed-off-by: Tibor Vass Conflicts: pkg/archive/archive.go fixed conflict which git couldn't fix with the added BreakoutError --- pkg/archive/archive.go | 5 + pkg/archive/archive_test.go | 192 +++++++++++++++++++++++++++++++++++- pkg/archive/diff_test.go | 191 +++++++++++++++++++++++++++++++++++ pkg/archive/utils_test.go | 166 +++++++++++++++++++++++++++++++ 4 files changed, 553 insertions(+), 1 deletion(-) create mode 100644 pkg/archive/diff_test.go create mode 100644 pkg/archive/utils_test.go diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index cdac22840..50ed8b72a 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -41,6 +41,11 @@ type ( Archiver struct { Untar func(io.Reader, string, *TarOptions) error } + + // breakoutError is used to differentiate errors related to breaking out + // When testing archive breakout in the unit tests, this error is expected + // in order for the test to pass. + breakoutError error ) var ( diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go index b46f95322..76dab21fa 100644 --- a/pkg/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "testing" "time" @@ -169,7 +170,12 @@ func TestTarWithOptions(t *testing.T) { // Failing prevents the archives from being uncompressed during ADD func TestTypeXGlobalHeaderDoesNotFail(t *testing.T) { hdr := tar.Header{Typeflag: tar.TypeXGlobalHeader} - err := createTarFile("pax_global_header", "some_dir", &hdr, nil, true) + tmpDir, err := ioutil.TempDir("", "docker-test-archive-pax-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + err = createTarFile(filepath.Join(tmpDir, "pax_global_header"), tmpDir, &hdr, nil, true) if err != nil { t.Fatal(err) } @@ -242,3 +248,187 @@ func BenchmarkTarUntar(b *testing.B) { os.RemoveAll(target) } } + +func TestUntarInvalidFilenames(t *testing.T) { + for i, headers := range [][]*tar.Header{ + { + { + Name: "../victim/dotdot", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + { + { + // Note the leading slash + Name: "/../victim/slash-dotdot", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + } { + if err := testBreakout("untar", "docker-TestUntarInvalidFilenames", headers); err != nil { + t.Fatalf("i=%d. %v", i, err) + } + } +} + +func TestUntarInvalidHardlink(t *testing.T) { + for i, headers := range [][]*tar.Header{ + { // try reading victim/hello (../) + { + Name: "dotdot", + Typeflag: tar.TypeLink, + Linkname: "../victim/hello", + Mode: 0644, + }, + }, + { // try reading victim/hello (/../) + { + Name: "slash-dotdot", + Typeflag: tar.TypeLink, + // Note the leading slash + Linkname: "/../victim/hello", + Mode: 0644, + }, + }, + { // try writing victim/file + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim/file", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + { // try reading victim/hello (hardlink, symlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // Try reading victim/hello (hardlink, hardlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "hardlink", + Typeflag: tar.TypeLink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // Try removing victim directory (hardlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + } { + if err := testBreakout("untar", "docker-TestUntarInvalidHardlink", headers); err != nil { + t.Fatalf("i=%d. %v", i, err) + } + } +} + +func TestUntarInvalidSymlink(t *testing.T) { + for i, headers := range [][]*tar.Header{ + { // try reading victim/hello (../) + { + Name: "dotdot", + Typeflag: tar.TypeSymlink, + Linkname: "../victim/hello", + Mode: 0644, + }, + }, + { // try reading victim/hello (/../) + { + Name: "slash-dotdot", + Typeflag: tar.TypeSymlink, + // Note the leading slash + Linkname: "/../victim/hello", + Mode: 0644, + }, + }, + { // try writing victim/file + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim/file", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + { // try reading victim/hello (symlink, symlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // try reading victim/hello (symlink, hardlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "hardlink", + Typeflag: tar.TypeLink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // try removing victim directory (symlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + } { + if err := testBreakout("untar", "docker-TestUntarInvalidSymlink", headers); err != nil { + t.Fatalf("i=%d. %v", i, err) + } + } +} diff --git a/pkg/archive/diff_test.go b/pkg/archive/diff_test.go new file mode 100644 index 000000000..758c4115d --- /dev/null +++ b/pkg/archive/diff_test.go @@ -0,0 +1,191 @@ +package archive + +import ( + "testing" + + "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" +) + +func TestApplyLayerInvalidFilenames(t *testing.T) { + for i, headers := range [][]*tar.Header{ + { + { + Name: "../victim/dotdot", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + { + { + // Note the leading slash + Name: "/../victim/slash-dotdot", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + } { + if err := testBreakout("applylayer", "docker-TestApplyLayerInvalidFilenames", headers); err != nil { + t.Fatalf("i=%d. %v", i, err) + } + } +} + +func TestApplyLayerInvalidHardlink(t *testing.T) { + for i, headers := range [][]*tar.Header{ + { // try reading victim/hello (../) + { + Name: "dotdot", + Typeflag: tar.TypeLink, + Linkname: "../victim/hello", + Mode: 0644, + }, + }, + { // try reading victim/hello (/../) + { + Name: "slash-dotdot", + Typeflag: tar.TypeLink, + // Note the leading slash + Linkname: "/../victim/hello", + Mode: 0644, + }, + }, + { // try writing victim/file + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim/file", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + { // try reading victim/hello (hardlink, symlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // Try reading victim/hello (hardlink, hardlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "hardlink", + Typeflag: tar.TypeLink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // Try removing victim directory (hardlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeLink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + } { + if err := testBreakout("applylayer", "docker-TestApplyLayerInvalidHardlink", headers); err != nil { + t.Fatalf("i=%d. %v", i, err) + } + } +} + +func TestApplyLayerInvalidSymlink(t *testing.T) { + for i, headers := range [][]*tar.Header{ + { // try reading victim/hello (../) + { + Name: "dotdot", + Typeflag: tar.TypeSymlink, + Linkname: "../victim/hello", + Mode: 0644, + }, + }, + { // try reading victim/hello (/../) + { + Name: "slash-dotdot", + Typeflag: tar.TypeSymlink, + // Note the leading slash + Linkname: "/../victim/hello", + Mode: 0644, + }, + }, + { // try writing victim/file + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim/file", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + { // try reading victim/hello (symlink, symlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // try reading victim/hello (symlink, hardlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "hardlink", + Typeflag: tar.TypeLink, + Linkname: "loophole-victim/hello", + Mode: 0644, + }, + }, + { // try removing victim directory (symlink) + { + Name: "loophole-victim", + Typeflag: tar.TypeSymlink, + Linkname: "../victim", + Mode: 0755, + }, + { + Name: "loophole-victim", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, + } { + if err := testBreakout("applylayer", "docker-TestApplyLayerInvalidSymlink", headers); err != nil { + t.Fatalf("i=%d. %v", i, err) + } + } +} diff --git a/pkg/archive/utils_test.go b/pkg/archive/utils_test.go new file mode 100644 index 000000000..3624fe5af --- /dev/null +++ b/pkg/archive/utils_test.go @@ -0,0 +1,166 @@ +package archive + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" +) + +var testUntarFns = map[string]func(string, io.Reader) error{ + "untar": func(dest string, r io.Reader) error { + return Untar(r, dest, nil) + }, + "applylayer": func(dest string, r io.Reader) error { + return ApplyLayer(dest, ArchiveReader(r)) + }, +} + +// testBreakout is a helper function that, within the provided `tmpdir` directory, +// creates a `victim` folder with a generated `hello` file in it. +// `untar` extracts to a directory named `dest`, the tar file created from `headers`. +// +// Here are the tested scenarios: +// - removed `victim` folder (write) +// - removed files from `victim` folder (write) +// - new files in `victim` folder (write) +// - modified files in `victim` folder (write) +// - file in `dest` with same content as `victim/hello` (read) +// +// When using testBreakout make sure you cover one of the scenarios listed above. +func testBreakout(untarFn string, tmpdir string, headers []*tar.Header) error { + tmpdir, err := ioutil.TempDir("", tmpdir) + if err != nil { + return err + } + defer os.RemoveAll(tmpdir) + + dest := filepath.Join(tmpdir, "dest") + if err := os.Mkdir(dest, 0755); err != nil { + return err + } + + victim := filepath.Join(tmpdir, "victim") + if err := os.Mkdir(victim, 0755); err != nil { + return err + } + hello := filepath.Join(victim, "hello") + helloData, err := time.Now().MarshalText() + if err != nil { + return err + } + if err := ioutil.WriteFile(hello, helloData, 0644); err != nil { + return err + } + helloStat, err := os.Stat(hello) + if err != nil { + return err + } + + reader, writer := io.Pipe() + go func() { + t := tar.NewWriter(writer) + for _, hdr := range headers { + t.WriteHeader(hdr) + } + t.Close() + }() + + untar := testUntarFns[untarFn] + if untar == nil { + return fmt.Errorf("could not find untar function %q in testUntarFns", untarFn) + } + if err := untar(dest, reader); err != nil { + if _, ok := err.(breakoutError); !ok { + // If untar returns an error unrelated to an archive breakout, + // then consider this an unexpected error and abort. + return err + } + // Here, untar detected the breakout. + // Let's move on verifying that indeed there was no breakout. + fmt.Printf("breakoutError: %v\n", err) + } + + // Check victim folder + f, err := os.Open(victim) + if err != nil { + // codepath taken if victim folder was removed + return fmt.Errorf("archive breakout: error reading %q: %v", victim, err) + } + defer f.Close() + + // Check contents of victim folder + // + // We are only interested in getting 2 files from the victim folder, because if all is well + // we expect only one result, the `hello` file. If there is a second result, it cannot + // hold the same name `hello` and we assume that a new file got created in the victim folder. + // That is enough to detect an archive breakout. + names, err := f.Readdirnames(2) + if err != nil { + // codepath taken if victim is not a folder + return fmt.Errorf("archive breakout: error reading directory content of %q: %v", victim, err) + } + for _, name := range names { + if name != "hello" { + // codepath taken if new file was created in victim folder + return fmt.Errorf("archive breakout: new file %q", name) + } + } + + // Check victim/hello + f, err = os.Open(hello) + if err != nil { + // codepath taken if read permissions were removed + return fmt.Errorf("archive breakout: could not lstat %q: %v", hello, err) + } + defer f.Close() + b, err := ioutil.ReadAll(f) + if err != nil { + return err + } + fi, err := f.Stat() + if err != nil { + return err + } + if helloStat.IsDir() != fi.IsDir() || + // TODO: cannot check for fi.ModTime() change + helloStat.Mode() != fi.Mode() || + helloStat.Size() != fi.Size() || + !bytes.Equal(helloData, b) { + // codepath taken if hello has been modified + return fmt.Errorf("archive breakout: file %q has been modified. Contents: expected=%q, got=%q. FileInfo: expected=%#v, got=%#v.", hello, helloData, b, helloStat, fi) + } + + // Check that nothing in dest/ has the same content as victim/hello. + // Since victim/hello was generated with time.Now(), it is safe to assume + // that any file whose content matches exactly victim/hello, managed somehow + // to access victim/hello. + return filepath.Walk(dest, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + if err != nil { + // skip directory if error + return filepath.SkipDir + } + // enter directory + return nil + } + if err != nil { + // skip file if error + return nil + } + b, err := ioutil.ReadFile(path) + if err != nil { + // Houston, we have a problem. Aborting (space)walk. + return err + } + if bytes.Equal(helloData, b) { + return fmt.Errorf("archive breakout: file %q has been accessed via %q", hello, path) + } + return nil + }) +} From a111eea20cad697bb58ebfbf455ee1670c7cf82c Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 20 Oct 2014 15:36:28 -0400 Subject: [PATCH 34/67] archive: prevent breakout in Untar Signed-off-by: Tibor Vass --- pkg/archive/archive.go | 22 +++++++++++++++++++++- pkg/symlink/fs.go | 4 +++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 50ed8b72a..c5255013a 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/promise" + "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/system" ) @@ -275,11 +276,23 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L } case tar.TypeLink: - if err := os.Link(filepath.Join(extractDir, hdr.Linkname), path); err != nil { + targetPath := filepath.Join(extractDir, hdr.Linkname) + // check for hardlink breakout + if !strings.HasPrefix(targetPath, extractDir) { + return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) + } + if err := os.Link(targetPath, path); err != nil { return err } case tar.TypeSymlink: + // check for symlink breakout + if _, err := symlink.FollowSymlinkInScope(filepath.Join(filepath.Dir(path), hdr.Linkname), extractDir); err != nil { + if _, ok := err.(symlink.ErrBreakout); ok { + return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) + } + return err + } if err := os.Symlink(hdr.Linkname, path); err != nil { return err } @@ -424,6 +437,8 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) // identity (uncompressed), gzip, bzip2, xz. // FIXME: specify behavior when target path exists vs. doesn't exist. func Untar(archive io.Reader, dest string, options *TarOptions) error { + dest = filepath.Clean(dest) + if options == nil { options = &TarOptions{} } @@ -461,6 +476,7 @@ loop: } // Normalize name, for safety and for a simple is-root check + // This keeps "../" as-is, but normalizes "/../" to "/" hdr.Name = filepath.Clean(hdr.Name) for _, exclude := range options.Excludes { @@ -481,7 +497,11 @@ loop: } } + // Prevent symlink breakout path := filepath.Join(dest, hdr.Name) + if !strings.HasPrefix(path, dest) { + return breakoutError(fmt.Errorf("%q is outside of %q", path, dest)) + } // If path exits we almost always just want to remove and replace it // The only exception is when it is a directory *and* the file from diff --git a/pkg/symlink/fs.go b/pkg/symlink/fs.go index 6ce99c6bd..09271ffc4 100644 --- a/pkg/symlink/fs.go +++ b/pkg/symlink/fs.go @@ -10,6 +10,8 @@ import ( const maxLoopCounter = 100 +type ErrBreakout error + // FollowSymlink will follow an existing link and scope it to the root // path provided. // The role of this function is to return an absolute path in the root @@ -34,7 +36,7 @@ func FollowSymlinkInScope(link, root string) (string, error) { } if !strings.HasPrefix(filepath.Dir(link), root) { - return "", fmt.Errorf("%s is not within %s", link, root) + return "", ErrBreakout(fmt.Errorf("%s is not within %s", link, root)) } prev := "/" From 04175d07634600f2a5cf43dd197eb2a926aba8b9 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 31 Oct 2014 13:18:39 -0400 Subject: [PATCH 35/67] archive: prevent breakout in ApplyLayer Signed-off-by: Tibor Vass --- pkg/archive/diff.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index 215f62ec0..5ed1a1d90 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -24,6 +24,8 @@ func mkdev(major int64, minor int64) uint32 { // ApplyLayer parses a diff in the standard layer format from `layer`, and // applies it to the directory `dest`. func ApplyLayer(dest string, layer ArchiveReader) error { + dest = filepath.Clean(dest) + // We need to be able to set any perms oldmask := syscall.Umask(0) defer syscall.Umask(oldmask) @@ -93,6 +95,12 @@ func ApplyLayer(dest string, layer ArchiveReader) error { path := filepath.Join(dest, hdr.Name) base := filepath.Base(path) + + // Prevent symlink breakout + if !strings.HasPrefix(path, dest) { + return breakoutError(fmt.Errorf("%q is outside of %q", path, dest)) + } + if strings.HasPrefix(base, ".wh.") { originalBase := base[len(".wh."):] originalPath := filepath.Join(filepath.Dir(path), originalBase) From 1527979e87d361539c2791d4e89602aea3cce174 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 17:37:44 -0500 Subject: [PATCH 36/67] registry: refactor registry.IsSecure calls into registry.NewEndpoint Signed-off-by: Tibor Vass Conflicts: registry/endpoint.go registry/endpoint_test.go registry/registry_test.go --- graph/pull.go | 4 +--- graph/push.go | 4 +--- registry/endpoint.go | 38 ++++++++++++++++++++++++-------------- registry/endpoint_test.go | 27 +++++++++++++++++++++++++++ registry/registry_test.go | 29 +++++++++++++++++++++++++++++ registry/service.go | 6 ++---- 6 files changed, 84 insertions(+), 24 deletions(-) create mode 100644 registry/endpoint_test.go diff --git a/graph/pull.go b/graph/pull.go index 897e0813a..c1db605b8 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -113,9 +113,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Error(err) } - secure := registry.IsSecure(hostname, s.insecureRegistries) - - endpoint, err := registry.NewEndpoint(hostname, secure) + endpoint, err := registry.NewEndpoint(hostname, s.insecureRegistries) if err != nil { return job.Error(err) } diff --git a/graph/push.go b/graph/push.go index 1d2bf7242..165b580f2 100644 --- a/graph/push.go +++ b/graph/push.go @@ -214,9 +214,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err) } - secure := registry.IsSecure(hostname, s.insecureRegistries) - - endpoint, err := registry.NewEndpoint(hostname, secure) + endpoint, err := registry.NewEndpoint(hostname, s.insecureRegistries) if err != nil { return job.Error(err) } diff --git a/registry/endpoint.go b/registry/endpoint.go index 6dd4e1f60..2eac41ce3 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -33,21 +33,15 @@ func scanForApiVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { - var ( - endpoint = Endpoint{secure: secure} - trimmedHostname string - err error - ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname - } - trimmedHostname, endpoint.Version = scanForApiVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) +func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { + endpoint, err := newEndpoint(hostname) if err != nil { return nil, err } + secure := isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure = secure + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { @@ -65,12 +59,28 @@ func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { endpoint.URL.Scheme = "http" _, err2 := endpoint.Ping() if err2 == nil { - return &endpoint, nil + return endpoint, nil } return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } + return endpoint, nil +} +func newEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint = Endpoint{secure: true} + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } return &endpoint, nil } @@ -141,9 +151,9 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return info, nil } -// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func IsSecure(hostname string, insecureRegistries []string) bool { +func isSecure(hostname string, insecureRegistries []string) bool { if hostname == IndexServerAddress() { return true } diff --git a/registry/endpoint_test.go b/registry/endpoint_test.go new file mode 100644 index 000000000..0ec1220d9 --- /dev/null +++ b/registry/endpoint_test.go @@ -0,0 +1,27 @@ +package registry + +import "testing" + +func TestEndpointParse(t *testing.T) { + testData := []struct { + str string + expected string + }{ + {IndexServerAddress(), IndexServerAddress()}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + } + for _, td := range testData { + e, err := newEndpoint(td.str) + if err != nil { + t.Errorf("%q: %s", td.str, err) + } + if e == nil { + t.Logf("something's fishy, endpoint for %q is nil", td.str) + continue + } + if e.String() != td.expected { + t.Errorf("expected %q, got %q", td.expected, e.String()) + } + } +} diff --git a/registry/registry_test.go b/registry/registry_test.go index c9a9fc81b..7e63ee92a 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -316,3 +316,32 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + addr string + insecureRegistries []string + expected bool + }{ + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + {"localhost", []string{"localhost:5000"}, false}, + {"localhost:5000", []string{"localhost:5000"}, false}, + {"localhost", []string{"example.com"}, false}, + {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, + {"localhost", []string{}, false}, + {"localhost:5000", []string{}, false}, + {"127.0.0.1", []string{}, false}, + {"localhost", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, + {"127.0.0.1:5000", []string{"example.com"}, false}, + } + for _, tt := range tests { + if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + } + } +} diff --git a/registry/service.go b/registry/service.go index 7051d9343..53e8278b0 100644 --- a/registry/service.go +++ b/registry/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) + endpoint, err := NewEndpoint(addr, s.insecureRegistries) if err != nil { return job.Error(err) } @@ -92,9 +92,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { return job.Error(err) } - secure := IsSecure(hostname, s.insecureRegistries) - - endpoint, err := NewEndpoint(hostname, secure) + endpoint, err := NewEndpoint(hostname, s.insecureRegistries) if err != nil { return job.Error(err) } From 6152460c1e4c058676476314320b17e54de0c621 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 20:08:59 -0500 Subject: [PATCH 37/67] Put mock registry address in insecureRegistries for unit tests Signed-off-by: Tibor Vass Conflicts: registry/registry_mock_test.go --- registry/registry_mock_test.go | 16 +++++++++++----- registry/registry_test.go | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/registry/registry_mock_test.go b/registry/registry_mock_test.go index 379dc78f4..fc2b46b9b 100644 --- a/registry/registry_mock_test.go +++ b/registry/registry_mock_test.go @@ -19,8 +19,9 @@ import ( ) var ( - testHttpServer *httptest.Server - testLayers = map[string]map[string]string{ + testHTTPServer *httptest.Server + insecureRegistries []string + testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", @@ -99,7 +100,12 @@ func init() { // /v2/ r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") - testHttpServer = httptest.NewServer(handlerAccessLog(r)) + testHTTPServer = httptest.NewServer(handlerAccessLog(r)) + URL, err := url.Parse(testHTTPServer.URL) + if err != nil { + panic(err) + } + insecureRegistries = []string{URL.Host} } func handlerAccessLog(handler http.Handler) http.Handler { @@ -111,7 +117,7 @@ func handlerAccessLog(handler http.Handler) http.Handler { } func makeURL(req string) string { - return testHttpServer.URL + req + return testHTTPServer.URL + req } func writeHeaders(w http.ResponseWriter) { @@ -301,7 +307,7 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { } func handlerImages(w http.ResponseWriter, r *http.Request) { - u, _ := url.Parse(testHttpServer.URL) + u, _ := url.Parse(testHTTPServer.URL) w.Header().Add("X-Docker-Endpoints", fmt.Sprintf("%s , %s ", u.Host, "test.example.com")) w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { diff --git a/registry/registry_test.go b/registry/registry_test.go index 7e63ee92a..aaba3f0a3 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -18,7 +18,7 @@ var ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/"), false) + endpoint, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } @@ -30,7 +30,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/"), false) + ep, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } From eb3738347ade64c2b14995c30b400bc6aa48b84e Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 13 Nov 2014 06:56:36 -0800 Subject: [PATCH 38/67] registry: parse INDEXSERVERADDRESS into a URL for easier check in isSecure Signed-off-by: Tibor Vass --- registry/auth.go | 10 ++++++++++ registry/endpoint.go | 14 ++++++-------- registry/endpoint_test.go | 2 +- registry/registry_test.go | 1 + 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/registry/auth.go b/registry/auth.go index 7c0709a47..dad58c163 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "path" "strings" @@ -27,8 +28,17 @@ const ( var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") + IndexServerURL *url.URL ) +func init() { + url, err := url.Parse(INDEXSERVER) + if err != nil { + panic(err) + } + IndexServerURL = url +} + type AuthConfig struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` diff --git a/registry/endpoint.go b/registry/endpoint.go index 2eac41ce3..eb0e9a1fa 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -34,21 +34,18 @@ func scanForApiVersion(hostname string) (string, APIVersion) { } func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname) + endpoint, err := newEndpoint(hostname, insecureRegistries) if err != nil { return nil, err } - secure := isSecure(endpoint.URL.Host, insecureRegistries) - endpoint.secure = secure - // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { //TODO: triggering highland build can be done there without "failing" - if secure { + if endpoint.secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) @@ -67,9 +64,9 @@ func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error return endpoint, nil } -func newEndpoint(hostname string) (*Endpoint, error) { +func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { var ( - endpoint = Endpoint{secure: true} + endpoint = Endpoint{} trimmedHostname string err error ) @@ -81,6 +78,7 @@ func newEndpoint(hostname string) (*Endpoint, error) { if err != nil { return nil, err } + endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) return &endpoint, nil } @@ -154,7 +152,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. func isSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { + if hostname == IndexServerURL.Host { return true } diff --git a/registry/endpoint_test.go b/registry/endpoint_test.go index 0ec1220d9..54105ec17 100644 --- a/registry/endpoint_test.go +++ b/registry/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str) + e, err := newEndpoint(td.str, insecureRegistries) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/registry/registry_test.go b/registry/registry_test.go index aaba3f0a3..dbedefe05 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -323,6 +323,7 @@ func TestIsSecure(t *testing.T) { insecureRegistries []string expected bool }{ + {IndexServerURL.Host, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, From 5e2d02ab73c06aec4ddeca6965dee3ef3cf6a04e Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 16:31:15 -0500 Subject: [PATCH 39/67] Add the possibility of specifying a subnet for --insecure-registry Signed-off-by: Tibor Vass Conflicts: registry/endpoint.go --- daemon/config.go | 10 +++- docs/sources/reference/commandline/cli.md | 2 +- registry/endpoint.go | 61 ++++++++++++++++++++--- registry/registry_mock_test.go | 26 ++++++++++ registry/registry_test.go | 19 ++++--- 5 files changed, 103 insertions(+), 15 deletions(-) diff --git a/daemon/config.go b/daemon/config.go index 9e8d08e2a..e45e73b99 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -56,7 +56,7 @@ func (config *Config) InstallFlags() { flag.StringVar(&config.BridgeIP, []string{"#bip", "-bip"}, "", "Use this CIDR notation address for the network bridge's IP, not compatible with -b") flag.StringVar(&config.BridgeIface, []string{"b", "-bridge"}, "", "Attach containers to a pre-existing network bridge\nuse 'none' to disable container networking") flag.StringVar(&config.FixedCIDR, []string{"-fixed-cidr"}, "", "IPv4 subnet for fixed IPs (ex: 10.20.0.0/16)\nthis subnet must be nested in the bridge subnet (which is defined by -b or --bip)") - opts.ListVar(&config.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback)") + opts.ListVar(&config.InsecureRegistries, []string{"-insecure-registry"}, "Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) (e.g., localhost:5000 or 10.20.0.0/16)") flag.BoolVar(&config.InterContainerCommunication, []string{"#icc", "-icc"}, true, "Enable inter-container communication") flag.StringVar(&config.GraphDriver, []string{"s", "-storage-driver"}, "", "Force the Docker runtime to use a specific storage driver") flag.StringVar(&config.ExecDriver, []string{"e", "-exec-driver"}, "native", "Force the Docker runtime to use a specific exec driver") @@ -68,6 +68,14 @@ func (config *Config) InstallFlags() { opts.IPListVar(&config.Dns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers") opts.DnsSearchListVar(&config.DnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains") opts.MirrorListVar(&config.Mirrors, []string{"-registry-mirror"}, "Specify a preferred Docker registry mirror") + + // Localhost is by default considered as an insecure registry + // This is a stop-gap for people who are running a private registry on localhost (especially on Boot2docker). + // + // TODO: should we deprecate this once it is easier for people to set up a TLS registry or change + // daemon flags on boot2docker? + // If so, do not forget to check the TODO in TestIsSecure + config.InsecureRegistries = append(config.InsecureRegistries, "127.0.0.0/8") } func GetDefaultNetworkMtu() int { diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index a57b4cc28..728f95cee 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -70,7 +70,7 @@ expect an integer, and they can only be specified once. -g, --graph="/var/lib/docker" Path to use as the root of the Docker runtime -H, --host=[] The socket(s) to bind to in daemon mode or connect to in client mode, specified using one or more tcp://host:port, unix:///path/to/socket, fd://* or fd://socketfd. --icc=true Enable inter-container communication - --insecure-registry=[] Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) + --insecure-registry=[] Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) (ex: localhost:5000 or 10.20.0.0/16) --ip=0.0.0.0 Default IP address to use when binding container ports --ip-forward=true Enable net.ipv4.ip_forward --ip-masq=true Enable IP masquerading for bridge's IP range diff --git a/registry/endpoint.go b/registry/endpoint.go index eb0e9a1fa..d65fd7e8a 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" "strings" @@ -11,6 +12,9 @@ import ( "github.com/docker/docker/pkg/log" ) +// for mocking in unit tests +var lookupIP = net.LookupIP + // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. func scanForApiVersion(hostname string) (string, APIVersion) { var ( @@ -78,7 +82,10 @@ func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error if err != nil { return nil, err } - endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure, err = isSecure(endpoint.URL.Host, insecureRegistries) + if err != nil { + return nil, err + } return &endpoint, nil } @@ -151,16 +158,56 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func isSecure(hostname string, insecureRegistries []string) bool { +// +// The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. +// If the subnet contains one of the IPs of the registry specified by hostname, the latter is considered +// insecure. +// +// hostname should be a URL.Host (`host:port` or `host`) +func isSecure(hostname string, insecureRegistries []string) (bool, error) { if hostname == IndexServerURL.Host { - return true + return true, nil } - for _, h := range insecureRegistries { - if hostname == h { - return false + host, _, err := net.SplitHostPort(hostname) + if err != nil { + // assume hostname is of the form `host` without the port and go on. + host = hostname + } + addrs, err := lookupIP(host) + if err != nil { + ip := net.ParseIP(host) + if ip == nil { + // if resolving `host` fails, error out, since host is to be net.Dial-ed anyway + return true, fmt.Errorf("issecure: could not resolve %q: %v", host, err) + } + addrs = []net.IP{ip} + } + if len(addrs) == 0 { + return true, fmt.Errorf("issecure: could not resolve %q", host) + } + + for _, addr := range addrs { + for _, r := range insecureRegistries { + // hostname matches insecure registry + if hostname == r { + return false, nil + } + + // now assume a CIDR was passed to --insecure-registry + _, ipnet, err := net.ParseCIDR(r) + if err != nil { + // if could not parse it as a CIDR, even after removing + // assume it's not a CIDR and go on with the next candidate + continue + } + + // check if the addr falls in the subnet + if ipnet.Contains(addr) { + return false, nil + } } } - return true + return true, nil } diff --git a/registry/registry_mock_test.go b/registry/registry_mock_test.go index fc2b46b9b..50724f0f9 100644 --- a/registry/registry_mock_test.go +++ b/registry/registry_mock_test.go @@ -2,9 +2,11 @@ package registry import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "net/url" @@ -80,6 +82,11 @@ var ( "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } + mockHosts = map[string][]net.IP{ + "": {net.ParseIP("0.0.0.0")}, + "localhost": {net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + "example.com": {net.ParseIP("42.42.42.42")}, + } ) func init() { @@ -106,6 +113,25 @@ func init() { panic(err) } insecureRegistries = []string{URL.Host} + + // override net.LookupIP + lookupIP = func(host string) ([]net.IP, error) { + if host == "127.0.0.1" { + // I believe in future Go versions this will fail, so let's fix it later + return net.LookupIP(host) + } + for h, addrs := range mockHosts { + if host == h { + return addrs, nil + } + for _, addr := range addrs { + if addr.String() == host { + return []net.IP{addr}, nil + } + } + } + return nil, errors.New("lookup: no such host") + } } func handlerAccessLog(handler http.Handler) http.Handler { diff --git a/registry/registry_test.go b/registry/registry_test.go index dbedefe05..1ffb44f31 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -330,19 +330,26 @@ func TestIsSecure(t *testing.T) { {"localhost:5000", []string{"localhost:5000"}, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, - {"localhost", []string{}, false}, - {"localhost:5000", []string{}, false}, - {"127.0.0.1", []string{}, false}, + {"localhost", nil, false}, + {"localhost:5000", nil, false}, + {"127.0.0.1", nil, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, - {"example.com", []string{}, true}, + {"example.com", nil, true}, {"example.com", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"example.com"}, false}, + {"example.com:5000", []string{"42.42.0.0/16"}, false}, + {"example.com", []string{"42.42.0.0/16"}, false}, + {"example.com:5000", []string{"42.42.42.42/8"}, false}, + {"127.0.0.1:5000", []string{"127.0.0.0/8"}, false}, + {"42.42.42.42:5000", []string{"42.1.1.1/8"}, false}, } for _, tt := range tests { - if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + // TODO: remove this once we remove localhost insecure by default + insecureRegistries := append(tt.insecureRegistries, "127.0.0.0/8") + if sec, err := isSecure(tt.addr, insecureRegistries); err != nil || sec != tt.expected { + t.Fatalf("isSecure failed for %q %v, expected %v got %v. Error: %v", tt.addr, insecureRegistries, tt.expected, sec, err) } } } From c650d17a266f3fec3c48d7e3c79afe13a6ec139a Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 14 Nov 2014 11:04:47 -0800 Subject: [PATCH 40/67] Rewrite documentation for insecure registries Signed-off-by: Tibor Vass Conflicts: docs/sources/reference/commandline/cli.md --- docs/sources/reference/commandline/cli.md | 46 ++++++++++++++++------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 728f95cee..6e6ccd69a 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -70,7 +70,7 @@ expect an integer, and they can only be specified once. -g, --graph="/var/lib/docker" Path to use as the root of the Docker runtime -H, --host=[] The socket(s) to bind to in daemon mode or connect to in client mode, specified using one or more tcp://host:port, unix:///path/to/socket, fd://* or fd://socketfd. --icc=true Enable inter-container communication - --insecure-registry=[] Enable insecure communication with specified registries (no certificate verification for HTTPS and enable HTTP fallback) (ex: localhost:5000 or 10.20.0.0/16) + --insecure-registry=[] Enable insecure communication with specified registries (disables certificate verification for HTTPS and enables HTTP fallback) (e.g., localhost:5000 or 10.20.0.0/16) --ip=0.0.0.0 Default IP address to use when binding container ports --ip-forward=true Enable net.ipv4.ip_forward --ip-masq=true Enable IP masquerading for bridge's IP range @@ -193,24 +193,44 @@ To set the DNS server for all Docker containers, use To set the DNS search domain for all Docker containers, use `docker -d --dns-search example.com`. +### Insecure registries + +Docker considers a private registry either secure or insecure. +In the rest of this section, *registry* is used for *private registry*, and `myregistry:5000` +is a placeholder example for a private registry. + +A secure registry uses TLS and a copy of its CA certificate is placed on the Docker host at +`/etc/docker/certs.d/myregistry:5000/ca.crt`. +An insecure registry is either not using TLS (i.e., listening on plain text HTTP), or is using +TLS with a CA certificate not known by the Docker daemon. The latter can happen when the +certificate was not found under `/etc/docker/certs.d/myregistry:5000/`, or if the certificate +verification failed (i.e., wrong CA). + +By default, Docker assumes all, but local (see local registries below), registries are secure. +Communicating with an insecure registry is not possible if Docker assumes that registry is secure. +In order to communicate with an insecure registry, the Docker daemon requires `--insecure-registry` +in one of the following two forms: + +* `--insecure-registry myregistry:5000` tells the Docker daemon that myregistry:5000 should be considered insecure. +* `--insecure-registry 10.1.0.0/16` tells the Docker daemon that all registries whose domain resolve to an IP address is part +of the subnet described by the CIDR syntax, should be considered insecure. + +The flag can be used multiple times to allow multiple registries to be marked as insecure. + +If an insecure registry is not marked as insecure, `docker pull`, `docker push`, and `docker search` +will result in an error message prompting the user to either secure or pass the `--insecure-registry` +flag to the Docker daemon as described above. + +Local registries, whose IP address falls in the 127.0.0.0/8 range, are automatically marked as insecure +as of Docker 1.3.2. It is not recommended to rely on this, as it may change in the future. + + ### Miscellaneous options IP masquerading uses address translation to allow containers without a public IP to talk to other machines on the Internet. This may interfere with some network topologies and can be disabled with --ip-masq=false. - -By default, Docker will assume all registries are secured via TLS with certificate verification -enabled. Prior versions of Docker used an auto fallback if a registry did not support TLS -(or if the TLS connection failed). This introduced the opportunity for Man In The Middle (MITM) -attacks, so as of Docker 1.3.1, the user must now specify the `--insecure-registry` daemon flag -for each insecure registry. An insecure registry is either not using TLS (i.e. plain text HTTP), -or is using TLS with a CA certificate not known by the Docker daemon (i.e. certification -verification disabled). For example, if there is a registry listening for HTTP at 127.0.0.1:5000, -as of Docker 1.3.1 you are required to specify `--insecure-registry 127.0.0.1:5000` when starting -the Docker daemon. - - Docker supports softlinks for the Docker data directory (`/var/lib/docker`) and for `/var/lib/docker/tmp`. The `DOCKER_TMPDIR` and the data directory can be set like this: From 87f59e3802a44aecdb8febfd34a9661e136ffcf4 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 23 Oct 2014 15:26:52 -0700 Subject: [PATCH 41/67] Skip V2 registry and immediately fallback to V1 when mirrors are provided Since V2 registry does not yet implement mirrors, when mirrors are given automatically fallback to V1 without checking V2 first. Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/pull.go b/graph/pull.go index c1db605b8..05d5ec765 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -137,7 +137,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { mirrors = s.mirrors } - if isOfficial || endpoint.Version == registry.APIVersion2 { + if len(mirrors) == 0 && (isOfficial || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { return job.Errorf("error updating trust base graph: %s", err) From 409f65bfd1bde5e45c68e2e1ecf68016ed91426b Mon Sep 17 00:00:00 2001 From: unclejack Date: Tue, 18 Nov 2014 23:33:13 +0200 Subject: [PATCH 42/67] pkg/chrootarchive: provide TMPDIR for ApplyLayer Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- pkg/chrootarchive/diff.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/chrootarchive/diff.go b/pkg/chrootarchive/diff.go index 2133200c6..2653aefe9 100644 --- a/pkg/chrootarchive/diff.go +++ b/pkg/chrootarchive/diff.go @@ -3,6 +3,7 @@ package chrootarchive import ( "flag" "fmt" + "io/ioutil" "os" "runtime" "syscall" @@ -21,9 +22,16 @@ func applyLayer() { if err := syscall.Chdir("/"); err != nil { fatal(err) } - if err := archive.ApplyLayer("/", os.Stdin); err != nil { + tmpDir, err := ioutil.TempDir("/", "temp-docker-extract") + if err != nil { fatal(err) } + os.Setenv("TMPDIR", tmpDir) + if err := archive.ApplyLayer("/", os.Stdin); err != nil { + os.RemoveAll(tmpDir) + fatal(err) + } + os.RemoveAll(tmpDir) os.Exit(0) } From b256616589700f895845e3c5aa72e9b4511cd67d Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 19 Nov 2014 11:27:34 -0500 Subject: [PATCH 43/67] archive: do not call FollowSymlinkInScope in createTarFile Signed-off-by: Tibor Vass --- pkg/archive/archive.go | 15 ++++++++------- pkg/archive/archive_test.go | 14 ++++++++++++++ pkg/symlink/fs.go | 4 +--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index c5255013a..155145f5d 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -22,7 +22,6 @@ import ( "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/promise" - "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/system" ) @@ -286,12 +285,14 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L } case tar.TypeSymlink: - // check for symlink breakout - if _, err := symlink.FollowSymlinkInScope(filepath.Join(filepath.Dir(path), hdr.Linkname), extractDir); err != nil { - if _, ok := err.(symlink.ErrBreakout); ok { - return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) - } - return err + // path -> hdr.Linkname = targetPath + // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file + targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) + + // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because + // that symlink would first have to be created, which would be caught earlier, at this very check: + if !strings.HasPrefix(targetPath, extractDir) { + return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) } if err := os.Symlink(hdr.Linkname, path); err != nil { return err diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go index 76dab21fa..7c9db4443 100644 --- a/pkg/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -426,6 +426,20 @@ func TestUntarInvalidSymlink(t *testing.T) { Mode: 0644, }, }, + { // try writing to victim/newdir/newfile with a symlink in the path + { + // this header needs to be before the next one, or else there is an error + Name: "dir/loophole", + Typeflag: tar.TypeSymlink, + Linkname: "../../victim", + Mode: 0755, + }, + { + Name: "dir/loophole/newdir/newfile", + Typeflag: tar.TypeReg, + Mode: 0644, + }, + }, } { if err := testBreakout("untar", "docker-TestUntarInvalidSymlink", headers); err != nil { t.Fatalf("i=%d. %v", i, err) diff --git a/pkg/symlink/fs.go b/pkg/symlink/fs.go index 09271ffc4..6ce99c6bd 100644 --- a/pkg/symlink/fs.go +++ b/pkg/symlink/fs.go @@ -10,8 +10,6 @@ import ( const maxLoopCounter = 100 -type ErrBreakout error - // FollowSymlink will follow an existing link and scope it to the root // path provided. // The role of this function is to return an absolute path in the root @@ -36,7 +34,7 @@ func FollowSymlinkInScope(link, root string) (string, error) { } if !strings.HasPrefix(filepath.Dir(link), root) { - return "", ErrBreakout(fmt.Errorf("%s is not within %s", link, root)) + return "", fmt.Errorf("%s is not within %s", link, root) } prev := "/" From 324953d74adf6803efd8796274a80ae0b0cbdbf8 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Mon, 24 Nov 2014 11:33:38 -0500 Subject: [PATCH 44/67] docs: Add 1.3.2 release notes Signed-off-by: Tibor Vass --- docs/sources/release-notes.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/sources/release-notes.md b/docs/sources/release-notes.md index 6b2f0fd57..a19eb71b6 100644 --- a/docs/sources/release-notes.md +++ b/docs/sources/release-notes.md @@ -4,6 +4,35 @@ understanding, release #Release Notes +##Version 1.3.2 +(2014-11-24) + +This release fixes some bugs and addresses some security issues. We have also +made improvements to aspects of `docker run`. + +*Security fixes* + +Patches and changes were made to address CVE-2014-6407 and CVE-2014-6408. +Specifically, changes were made in order to: + +* Prevent host privilege escalation from an image extraction vulnerability (CVE-2014-6407). + +* Prevent container escalation from malicious security options applied to images (CVE-2014-6408). + +*Daemon fixes* + +The `--insecure-registry` flag of the `docker run` command has undergone +several refinements and additions. For details, please see the +[command-line reference](http://docs.docker.com/reference/commandline/cli/#run). + +* You can now specify a sub-net in order to set a range of registries which the Docker daemon will consider insecure. + +* By default, Docker now defines `localhost` as an insecure registry. + +* Registries can now be referenced using the Classless Inter-Domain Routing (CIDR) format. + +* When mirroring is enabled, the experimental registry v2 API is skipped. + ##Version 1.3.1 (2014-10-28) From 39fa2faad2f3d6fa5133de4eb740677202f53ef4 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 14 Nov 2014 14:47:37 -0800 Subject: [PATCH 45/67] Bump to version v1.3.2 Signed-off-by: Tibor Vass --- CHANGELOG.md | 16 ++++++++++++++++ VERSION | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f958bbd48..2cddc6c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 1.3.2 (2014-11-20) + +#### Security +- Fix tar breakout vulnerability +* Extractions are now sandboxed chroot +- Security options are no longer committed to images + +#### Runtime +- Fix deadlock in `docker ps -f exited=1` +- Fix a bug when `--volumes-from` references a container that failed to start + +#### Registry ++ `--insecure-registry` now accepts CIDR notation such as 10.1.0.0/16 +* Private registries whose IPs fall in the 127.0.0.0/8 range do no need the `--insecure-registry` flag +- Skip the experimental registry v2 API when mirroring is enabled + ## 1.3.1 (2014-10-28) #### Security diff --git a/VERSION b/VERSION index 3a3cd8cc8..1892b9267 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.1 +1.3.2 From cd745d5c6e012ff603f25e1314cfb4c0b09e906c Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 5 Dec 2014 14:58:46 -0700 Subject: [PATCH 46/67] Simplify FollowSymlinkInScope based on Go 1.3.3's EvalSymlinks Signed-off-by: Andrew Page --- pkg/symlink/LICENSE.APACHE | 191 +++++++++++++++++++++++++++++++++++++ pkg/symlink/LICENSE.BSD | 27 ++++++ pkg/symlink/MAINTAINERS | 5 +- pkg/symlink/README.md | 5 + pkg/symlink/fs.go | 182 ++++++++++++++++++++--------------- pkg/symlink/fs_test.go | 2 + 6 files changed, 334 insertions(+), 78 deletions(-) create mode 100644 pkg/symlink/LICENSE.APACHE create mode 100644 pkg/symlink/LICENSE.BSD create mode 100644 pkg/symlink/README.md diff --git a/pkg/symlink/LICENSE.APACHE b/pkg/symlink/LICENSE.APACHE new file mode 100644 index 000000000..27448585a --- /dev/null +++ b/pkg/symlink/LICENSE.APACHE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2014 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pkg/symlink/LICENSE.BSD b/pkg/symlink/LICENSE.BSD new file mode 100644 index 000000000..ebcfbcc77 --- /dev/null +++ b/pkg/symlink/LICENSE.BSD @@ -0,0 +1,27 @@ +Copyright (c) 2014 The Docker & Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/symlink/MAINTAINERS b/pkg/symlink/MAINTAINERS index 68a97d2fc..51a41a5b6 100644 --- a/pkg/symlink/MAINTAINERS +++ b/pkg/symlink/MAINTAINERS @@ -1,2 +1,3 @@ -Michael Crosby (@crosbymichael) -Victor Vieux (@vieux) +Tibor Vass (@tiborvass) +Cristian Staretu (@unclejack) +Tianon Gravi (@tianon) diff --git a/pkg/symlink/README.md b/pkg/symlink/README.md new file mode 100644 index 000000000..0d1dbb70e --- /dev/null +++ b/pkg/symlink/README.md @@ -0,0 +1,5 @@ +Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks +from the [Go standard library](https://golang.org/pkg/path/filepath). + +The code from filepath.EvalSymlinks has been adapted in fs.go. +Please read the LICENSE.BSD file that governs fs.go and LICENSE.APACHE for fs_test.go. diff --git a/pkg/symlink/fs.go b/pkg/symlink/fs.go index 6ce99c6bd..b4bdff24d 100644 --- a/pkg/symlink/fs.go +++ b/pkg/symlink/fs.go @@ -1,101 +1,131 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE.BSD file. + +// This code is a modified version of path/filepath/symlink.go from the Go standard library. + package symlink import ( - "fmt" + "bytes" + "errors" "os" - "path" "path/filepath" "strings" ) -const maxLoopCounter = 100 - -// FollowSymlink will follow an existing link and scope it to the root -// path provided. -// The role of this function is to return an absolute path in the root -// or normalize to the root if the symlink leads to a path which is -// outside of the root. -// Errors encountered while attempting to follow the symlink in path -// will be reported. -// Normalizations to the root don't constitute errors. -func FollowSymlinkInScope(link, root string) (string, error) { - root, err := filepath.Abs(root) +// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an absolute path +func FollowSymlinkInScope(path, root string) (string, error) { + path, err := filepath.Abs(path) if err != nil { return "", err } - - link, err = filepath.Abs(link) + root, err = filepath.Abs(root) if err != nil { return "", err } + return evalSymlinksInScope(path, root) +} - if link == root { - return root, nil +// evalSymlinksInScope will evaluate symlinks in `path` within a scope `root` and return +// a result guaranteed to be contained within the scope `root`, at the time of the call. +// Symlinks in `root` are not evaluated and left as-is. +// Errors encountered while attempting to evaluate symlinks in path will be returned. +// Non-existing paths are valid and do not constitute an error. +// `path` has to contain `root` as a prefix, or else an error will be returned. +// Trying to break out from `root` does not constitute an error. +// +// Example: +// If /foo/bar -> /outside, +// FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/oustide" +// +// IMPORTANT: it is the caller's responsibility to call evalSymlinksInScope *after* relevant symlinks +// are created and not to create subsequently, additional symlinks that could potentially make a +// previously-safe path, unsafe. Example: if /foo/bar does not exist, evalSymlinksInScope("/foo/bar", "/foo") +// would return "/foo/bar". If one makes /foo/bar a symlink to /baz subsequently, then "/foo/bar" should +// no longer be considered safely contained in "/foo". +func evalSymlinksInScope(path, root string) (string, error) { + root = filepath.Clean(root) + if path == root { + return path, nil } - - if !strings.HasPrefix(filepath.Dir(link), root) { - return "", fmt.Errorf("%s is not within %s", link, root) + if !strings.HasPrefix(path, root) { + return "", errors.New("evalSymlinksInScope: " + path + " is not in " + root) + } + const maxIter = 255 + originalPath := path + // given root of "/a" and path of "/a/b/../../c" we want path to be "/b/../../c" + path = path[len(root):] + if root == string(filepath.Separator) { + path = string(filepath.Separator) + path + } + if !strings.HasPrefix(path, string(filepath.Separator)) { + return "", errors.New("evalSymlinksInScope: " + path + " is not in " + root) } + path = filepath.Clean(path) + // consume path by taking each frontmost path element, + // expanding it if it's a symlink, and appending it to b + var b bytes.Buffer + // b here will always be considered to be the "current absolute path inside + // root" when we append paths to it, we also append a slash and use + // filepath.Clean after the loop to trim the trailing slash + for n := 0; path != ""; n++ { + if n > maxIter { + return "", errors.New("evalSymlinksInScope: too many links in " + originalPath) + } - prev := "/" + // find next path component, p + i := strings.IndexRune(path, filepath.Separator) + var p string + if i == -1 { + p, path = path, "" + } else { + p, path = path[:i], path[i+1:] + } - for _, p := range strings.Split(link, "/") { - prev = filepath.Join(prev, p) + if p == "" { + continue + } - loopCounter := 0 - for { - loopCounter++ + // this takes a b.String() like "b/../" and a p like "c" and turns it + // into "/b/../c" which then gets filepath.Cleaned into "/c" and then + // root gets prepended and we Clean again (to remove any trailing slash + // if the first Clean gave us just "/") + cleanP := filepath.Clean(string(filepath.Separator) + b.String() + p) + if cleanP == string(filepath.Separator) { + // never Lstat "/" itself + b.Reset() + continue + } + fullP := filepath.Clean(root + cleanP) - if loopCounter >= maxLoopCounter { - return "", fmt.Errorf("loopCounter reached MAX: %v", loopCounter) - } + fi, err := os.Lstat(fullP) + if os.IsNotExist(err) { + // if p does not exist, accept it + b.WriteString(p) + b.WriteRune(filepath.Separator) + continue + } + if err != nil { + return "", err + } + if fi.Mode()&os.ModeSymlink == 0 { + b.WriteString(p + string(filepath.Separator)) + continue + } - if !strings.HasPrefix(prev, root) { - // Don't resolve symlinks outside of root. For example, - // we don't have to check /home in the below. - // - // /home -> usr/home - // FollowSymlinkInScope("/home/bob/foo/bar", "/home/bob/foo") - break - } - - stat, err := os.Lstat(prev) - if err != nil { - if os.IsNotExist(err) { - break - } - return "", err - } - - // let's break if we're not dealing with a symlink - if stat.Mode()&os.ModeSymlink != os.ModeSymlink { - break - } - - // process the symlink - dest, err := os.Readlink(prev) - if err != nil { - return "", err - } - - if path.IsAbs(dest) { - prev = filepath.Join(root, dest) - } else { - prev, _ = filepath.Abs(prev) - - dir := filepath.Dir(prev) - prev = filepath.Join(dir, dest) - if dir == root && !strings.HasPrefix(prev, root) { - prev = root - } - if len(prev) < len(root) || (len(prev) == len(root) && prev != root) { - prev = filepath.Join(root, filepath.Base(dest)) - } - } + // it's a symlink, put it at the front of path + dest, err := os.Readlink(fullP) + if err != nil { + return "", err + } + if filepath.IsAbs(dest) { + b.Reset() } + path = dest + string(filepath.Separator) + path } - if prev == "/" { - prev = root - } - return prev, nil + + // see note above on "fullP := ..." for why this is double-cleaned and + // what's happening here + return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil } diff --git a/pkg/symlink/fs_test.go b/pkg/symlink/fs_test.go index 0e2f948b6..24ffb1e7c 100644 --- a/pkg/symlink/fs_test.go +++ b/pkg/symlink/fs_test.go @@ -1,3 +1,5 @@ +// Licensed under the Apache License, Version 2.0; See LICENSE.APACHE + package symlink import ( From 18193ae0a3a1fb506e7855c5e2e02c2c2d4c1a3e Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Sun, 30 Nov 2014 11:39:28 -0500 Subject: [PATCH 47/67] Refactor of symlink tests to remove testdata dir Signed-off-by: Tibor Vass --- pkg/symlink/fs_test.go | 295 ++++++++++++++++-------------------- pkg/symlink/testdata/fs/a/d | 1 - pkg/symlink/testdata/fs/a/e | 1 - pkg/symlink/testdata/fs/a/f | 1 - pkg/symlink/testdata/fs/b/h | 1 - pkg/symlink/testdata/fs/g | 1 - pkg/symlink/testdata/fs/i | 1 - pkg/symlink/testdata/fs/j/k | 1 - 8 files changed, 132 insertions(+), 170 deletions(-) delete mode 120000 pkg/symlink/testdata/fs/a/d delete mode 120000 pkg/symlink/testdata/fs/a/e delete mode 120000 pkg/symlink/testdata/fs/a/f delete mode 120000 pkg/symlink/testdata/fs/b/h delete mode 120000 pkg/symlink/testdata/fs/g delete mode 120000 pkg/symlink/testdata/fs/i delete mode 120000 pkg/symlink/testdata/fs/j/k diff --git a/pkg/symlink/fs_test.go b/pkg/symlink/fs_test.go index 24ffb1e7c..9d12041dc 100644 --- a/pkg/symlink/fs_test.go +++ b/pkg/symlink/fs_test.go @@ -3,248 +3,217 @@ package symlink import ( + "fmt" "io/ioutil" "os" "path/filepath" "testing" ) -func abs(t *testing.T, p string) string { - o, err := filepath.Abs(p) - if err != nil { - t.Fatal(err) - } - return o +type dirOrLink struct { + path string + target string } -func TestFollowSymLinkNormal(t *testing.T) { - link := "testdata/fs/a/d/c/data" +func makeFs(tmpdir string, fs []dirOrLink) error { + for _, s := range fs { + s.path = filepath.Join(tmpdir, s.path) + if s.target == "" { + os.MkdirAll(s.path, 0755) + continue + } + if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { + return err + } + if err := os.Symlink(s.target, s.path); err != nil && !os.IsExist(err) { + return err + } + } + return nil +} - rewrite, err := FollowSymlinkInScope(link, "testdata") +func testSymlink(tmpdir, path, expected, scope string) error { + rewrite, err := FollowSymlinkInScope(filepath.Join(tmpdir, path), filepath.Join(tmpdir, scope)) + if err != nil { + return err + } + expected, err = filepath.Abs(filepath.Join(tmpdir, expected)) + if err != nil { + return err + } + if expected != rewrite { + return fmt.Errorf("Expected %q got %q", expected, rewrite) + } + return nil +} + +func TestFollowSymlinkNormal(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkNormal") if err != nil { t.Fatal(err) } - - if expected := abs(t, "testdata/b/c/data"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) + defer os.RemoveAll(tmpdir) + if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/d", target: "/b"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/a/d/c/data", "testdata/b/c/data", "testdata"); err != nil { + t.Fatal(err) } } -func TestFollowSymLinkRelativePath(t *testing.T) { - link := "testdata/fs/i" - - rewrite, err := FollowSymlinkInScope(link, "testdata") +func TestFollowSymlinkRelativePath(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativePath") if err != nil { t.Fatal(err) } - - if expected := abs(t, "testdata/fs/a"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) + defer os.RemoveAll(tmpdir) + if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/i", target: "a"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/i", "testdata/fs/a", "testdata"); err != nil { + t.Fatal(err) } } -func TestFollowSymLinkUnderLinkedDir(t *testing.T) { - dir, err := ioutil.TempDir("", "docker-fs-test") +func TestFollowSymlinkUnderLinkedDir(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkUnderLinkedDir") if err != nil { t.Fatal(err) } - defer os.RemoveAll(dir) - - os.Mkdir(filepath.Join(dir, "realdir"), 0700) - os.Symlink("realdir", filepath.Join(dir, "linkdir")) - - linkDir := filepath.Join(dir, "linkdir", "foo") - dirUnderLinkDir := filepath.Join(dir, "linkdir", "foo", "bar") - os.MkdirAll(dirUnderLinkDir, 0700) - - rewrite, err := FollowSymlinkInScope(dirUnderLinkDir, linkDir) - if err != nil { + defer os.RemoveAll(tmpdir) + if err := makeFs(tmpdir, []dirOrLink{ + {path: "linkdir", target: "realdir"}, + {path: "linkdir/foo/bar"}, + }); err != nil { t.Fatal(err) } - - if rewrite != dirUnderLinkDir { - t.Fatalf("Expected %s got %s", dirUnderLinkDir, rewrite) + if err := testSymlink(tmpdir, "linkdir/foo/bar", "linkdir/foo/bar", "linkdir/foo"); err != nil { + t.Fatal(err) } } -func TestFollowSymLinkRandomString(t *testing.T) { +func TestFollowSymlinkRandomString(t *testing.T) { if _, err := FollowSymlinkInScope("toto", "testdata"); err == nil { t.Fatal("Random string should fail but didn't") } } -func TestFollowSymLinkLastLink(t *testing.T) { - link := "testdata/fs/a/d" - - rewrite, err := FollowSymlinkInScope(link, "testdata") +func TestFollowSymlinkLastLink(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkLastLink") if err != nil { t.Fatal(err) } - - if expected := abs(t, "testdata/b"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) + defer os.RemoveAll(tmpdir) + if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/d", target: "/b"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/a/d", "testdata/b", "testdata"); err != nil { + t.Fatal(err) } } -func TestFollowSymLinkRelativeLink(t *testing.T) { - link := "testdata/fs/a/e/c/data" - - rewrite, err := FollowSymlinkInScope(link, "testdata") +func TestFollowSymlinkRelativeLink(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLink") if err != nil { t.Fatal(err) } - - if expected := abs(t, "testdata/fs/b/c/data"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) + defer os.RemoveAll(tmpdir) + if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/e", target: "../b"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/a/e/c/data", "testdata/fs/b/c/data", "testdata"); err != nil { + t.Fatal(err) } } -func TestFollowSymLinkRelativeLinkScope(t *testing.T) { +func TestFollowSymlinkRelativeLinkScope(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkScope") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/f", target: "../../../../test"}}); err != nil { + t.Fatal(err) + } // avoid letting symlink f lead us out of the "testdata" scope // we don't normalize because symlink f is in scope and there is no // information leak - { - link := "testdata/fs/a/f" - - rewrite, err := FollowSymlinkInScope(link, "testdata") - if err != nil { - t.Fatal(err) - } - - if expected := abs(t, "testdata/test"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) - } + if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/test", "testdata"); err != nil { + t.Fatal(err) } - // avoid letting symlink f lead us out of the "testdata/fs" scope // we don't normalize because symlink f is in scope and there is no // information leak - { - link := "testdata/fs/a/f" - - rewrite, err := FollowSymlinkInScope(link, "testdata/fs") - if err != nil { - t.Fatal(err) - } - - if expected := abs(t, "testdata/fs/test"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) - } + if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/fs/test", "testdata/fs"); err != nil { + t.Fatal(err) } // avoid letting symlink g (pointed at by symlink h) take out of scope // TODO: we should probably normalize to scope here because ../[....]/root // is out of scope and we leak information - { - link := "testdata/fs/b/h" - - rewrite, err := FollowSymlinkInScope(link, "testdata") - if err != nil { - t.Fatal(err) - } - - if expected := abs(t, "testdata/root"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) - } + if err := makeFs(tmpdir, []dirOrLink{ + {path: "testdata/fs/b/h", target: "../g"}, + {path: "testdata/fs/g", target: "../../../../../../../../../../../../root"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/b/h", "testdata/root", "testdata"); err != nil { + t.Fatal(err) } // avoid letting allowing symlink e lead us to ../b // normalize to the "testdata/fs/a" - { - link := "testdata/fs/a/e" - - rewrite, err := FollowSymlinkInScope(link, "testdata/fs/a") - if err != nil { - t.Fatal(err) - } - - if expected := abs(t, "testdata/fs/a"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) - } + if err := makeFs(tmpdir, []dirOrLink{ + {path: "testdata/fs/a/e", target: "../b"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/a/e", "testdata/fs/a/b", "testdata/fs/a"); err != nil { + t.Fatal(err) } // avoid letting symlink -> ../directory/file escape from scope // normalize to "testdata/fs/j" - { - link := "testdata/fs/j/k" - - rewrite, err := FollowSymlinkInScope(link, "testdata/fs/j") - if err != nil { - t.Fatal(err) - } - - if expected := abs(t, "testdata/fs/j"); expected != rewrite { - t.Fatalf("Expected %s got %s", expected, rewrite) - } + if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/j/k", target: "../i/a"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "testdata/fs/j/k", "testdata/fs/j/i/a", "testdata/fs/j"); err != nil { + t.Fatal(err) } // make sure we don't allow escaping to / // normalize to dir - { - dir, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - linkFile := filepath.Join(dir, "foo") - os.Mkdir(filepath.Join(dir, ""), 0700) - os.Symlink("/", linkFile) - - rewrite, err := FollowSymlinkInScope(linkFile, dir) - if err != nil { - t.Fatal(err) - } - - if rewrite != dir { - t.Fatalf("Expected %s got %s", dir, rewrite) - } + if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "/"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "foo", "", ""); err != nil { + t.Fatal(err) } // make sure we don't allow escaping to / // normalize to dir - { - dir, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - linkFile := filepath.Join(dir, "foo") - os.Mkdir(filepath.Join(dir, ""), 0700) - os.Symlink("/../../", linkFile) - - rewrite, err := FollowSymlinkInScope(linkFile, dir) - if err != nil { - t.Fatal(err) - } - - if rewrite != dir { - t.Fatalf("Expected %s got %s", dir, rewrite) - } + if err := makeFs(filepath.Join(tmpdir, "dir", "subdir"), []dirOrLink{{path: "foo", target: "/../../"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "foo", "", ""); err != nil { + t.Fatal(err) } // make sure we stay in scope without leaking information // this also checks for escaping to / // normalize to dir - { - dir, err := ioutil.TempDir("", "docker-fs-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) + if err := makeFs(filepath.Join(tmpdir, "dir", "subdir"), []dirOrLink{{path: "foo", target: "../../"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "foo", "", ""); err != nil { + t.Fatal(err) + } - linkFile := filepath.Join(dir, "foo") - os.Mkdir(filepath.Join(dir, ""), 0700) - os.Symlink("../../", linkFile) - - rewrite, err := FollowSymlinkInScope(linkFile, dir) - if err != nil { - t.Fatal(err) - } - - if rewrite != dir { - t.Fatalf("Expected %s got %s", dir, rewrite) - } + if err := makeFs(tmpdir, []dirOrLink{{path: "bar/foo", target: "baz/target"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "bar/foo", "bar/baz/target", ""); err != nil { + t.Fatal(err) } } diff --git a/pkg/symlink/testdata/fs/a/d b/pkg/symlink/testdata/fs/a/d deleted file mode 120000 index 28abc9604..000000000 --- a/pkg/symlink/testdata/fs/a/d +++ /dev/null @@ -1 +0,0 @@ -/b \ No newline at end of file diff --git a/pkg/symlink/testdata/fs/a/e b/pkg/symlink/testdata/fs/a/e deleted file mode 120000 index 42532fe13..000000000 --- a/pkg/symlink/testdata/fs/a/e +++ /dev/null @@ -1 +0,0 @@ -../b \ No newline at end of file diff --git a/pkg/symlink/testdata/fs/a/f b/pkg/symlink/testdata/fs/a/f deleted file mode 120000 index 21de7edc0..000000000 --- a/pkg/symlink/testdata/fs/a/f +++ /dev/null @@ -1 +0,0 @@ -../../../../test \ No newline at end of file diff --git a/pkg/symlink/testdata/fs/b/h b/pkg/symlink/testdata/fs/b/h deleted file mode 120000 index 24387a68f..000000000 --- a/pkg/symlink/testdata/fs/b/h +++ /dev/null @@ -1 +0,0 @@ -../g \ No newline at end of file diff --git a/pkg/symlink/testdata/fs/g b/pkg/symlink/testdata/fs/g deleted file mode 120000 index 0ce5de064..000000000 --- a/pkg/symlink/testdata/fs/g +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../../../../../../root \ No newline at end of file diff --git a/pkg/symlink/testdata/fs/i b/pkg/symlink/testdata/fs/i deleted file mode 120000 index 2e65efe2a..000000000 --- a/pkg/symlink/testdata/fs/i +++ /dev/null @@ -1 +0,0 @@ -a \ No newline at end of file diff --git a/pkg/symlink/testdata/fs/j/k b/pkg/symlink/testdata/fs/j/k deleted file mode 120000 index f559e8fda..000000000 --- a/pkg/symlink/testdata/fs/j/k +++ /dev/null @@ -1 +0,0 @@ -../i/a \ No newline at end of file From 1ebafeb635eead4d3eedf4b5bbaf7e1e2d760086 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 5 Dec 2014 15:33:11 -0500 Subject: [PATCH 48/67] symlink: cleanup names and break big test into multiple smaller ones Signed-off-by: Tibor Vass --- pkg/symlink/fs_test.go | 84 ++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 20 deletions(-) diff --git a/pkg/symlink/fs_test.go b/pkg/symlink/fs_test.go index 9d12041dc..3869e1d91 100644 --- a/pkg/symlink/fs_test.go +++ b/pkg/symlink/fs_test.go @@ -47,8 +47,8 @@ func testSymlink(tmpdir, path, expected, scope string) error { return nil } -func TestFollowSymlinkNormal(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkNormal") +func TestFollowSymlinkAbsolute(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkAbsolute") if err != nil { t.Fatal(err) } @@ -75,8 +75,8 @@ func TestFollowSymlinkRelativePath(t *testing.T) { } } -func TestFollowSymlinkUnderLinkedDir(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkUnderLinkedDir") +func TestFollowSymlinkSkipSymlinksOutsideScope(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkSkipSymlinksOutsideScope") if err != nil { t.Fatal(err) } @@ -92,9 +92,9 @@ func TestFollowSymlinkUnderLinkedDir(t *testing.T) { } } -func TestFollowSymlinkRandomString(t *testing.T) { +func TestFollowSymlinkInvalidScopePathPair(t *testing.T) { if _, err := FollowSymlinkInScope("toto", "testdata"); err == nil { - t.Fatal("Random string should fail but didn't") + t.Fatal("expected an error") } } @@ -112,8 +112,8 @@ func TestFollowSymlinkLastLink(t *testing.T) { } } -func TestFollowSymlinkRelativeLink(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLink") +func TestFollowSymlinkRelativeLinkChangeScope(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkChangeScope") if err != nil { t.Fatal(err) } @@ -124,10 +124,15 @@ func TestFollowSymlinkRelativeLink(t *testing.T) { if err := testSymlink(tmpdir, "testdata/fs/a/e/c/data", "testdata/fs/b/c/data", "testdata"); err != nil { t.Fatal(err) } + // avoid letting allowing symlink e lead us to ../b + // normalize to the "testdata/fs/a" + if err := testSymlink(tmpdir, "testdata/fs/a/e", "testdata/fs/a/b", "testdata/fs/a"); err != nil { + t.Fatal(err) + } } -func TestFollowSymlinkRelativeLinkScope(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkScope") +func TestFollowSymlinkDeepRelativeLinkChangeScope(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkDeepRelativeLinkChangeScope") if err != nil { t.Fatal(err) } @@ -148,6 +153,14 @@ func TestFollowSymlinkRelativeLinkScope(t *testing.T) { if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/fs/test", "testdata/fs"); err != nil { t.Fatal(err) } +} + +func TestFollowSymlinkRelativeLinkChain(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkChain") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) // avoid letting symlink g (pointed at by symlink h) take out of scope // TODO: we should probably normalize to scope here because ../[....]/root @@ -161,17 +174,14 @@ func TestFollowSymlinkRelativeLinkScope(t *testing.T) { if err := testSymlink(tmpdir, "testdata/fs/b/h", "testdata/root", "testdata"); err != nil { t.Fatal(err) } +} - // avoid letting allowing symlink e lead us to ../b - // normalize to the "testdata/fs/a" - if err := makeFs(tmpdir, []dirOrLink{ - {path: "testdata/fs/a/e", target: "../b"}, - }); err != nil { - t.Fatal(err) - } - if err := testSymlink(tmpdir, "testdata/fs/a/e", "testdata/fs/a/b", "testdata/fs/a"); err != nil { +func TestFollowSymlinkBreakoutPath(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkBreakoutPath") + if err != nil { t.Fatal(err) } + defer os.RemoveAll(tmpdir) // avoid letting symlink -> ../directory/file escape from scope // normalize to "testdata/fs/j" @@ -181,6 +191,14 @@ func TestFollowSymlinkRelativeLinkScope(t *testing.T) { if err := testSymlink(tmpdir, "testdata/fs/j/k", "testdata/fs/j/i/a", "testdata/fs/j"); err != nil { t.Fatal(err) } +} + +func TestFollowSymlinkToRoot(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkToRoot") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) // make sure we don't allow escaping to / // normalize to dir @@ -190,25 +208,51 @@ func TestFollowSymlinkRelativeLinkScope(t *testing.T) { if err := testSymlink(tmpdir, "foo", "", ""); err != nil { t.Fatal(err) } +} + +func TestFollowSymlinkSlashDotdot(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkSlashDotdot") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + tmpdir = filepath.Join(tmpdir, "dir", "subdir") // make sure we don't allow escaping to / // normalize to dir - if err := makeFs(filepath.Join(tmpdir, "dir", "subdir"), []dirOrLink{{path: "foo", target: "/../../"}}); err != nil { + if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "/../../"}}); err != nil { t.Fatal(err) } if err := testSymlink(tmpdir, "foo", "", ""); err != nil { t.Fatal(err) } +} + +func TestFollowSymlinkDotdot(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkDotdot") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + tmpdir = filepath.Join(tmpdir, "dir", "subdir") // make sure we stay in scope without leaking information // this also checks for escaping to / // normalize to dir - if err := makeFs(filepath.Join(tmpdir, "dir", "subdir"), []dirOrLink{{path: "foo", target: "../../"}}); err != nil { + if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "../../"}}); err != nil { t.Fatal(err) } if err := testSymlink(tmpdir, "foo", "", ""); err != nil { t.Fatal(err) } +} + +func TestFollowSymlinkRelativePath2(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativePath2") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) if err := makeFs(tmpdir, []dirOrLink{{path: "bar/foo", target: "baz/target"}}); err != nil { t.Fatal(err) From 566146bc13efb6f8b1e6d4a41382f80546ac1782 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 5 Dec 2014 16:40:53 -0500 Subject: [PATCH 49/67] symlink: add more tests Signed-off-by: Tibor Vass --- pkg/symlink/fs_test.go | 139 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/pkg/symlink/fs_test.go b/pkg/symlink/fs_test.go index 3869e1d91..6b2496c4e 100644 --- a/pkg/symlink/fs_test.go +++ b/pkg/symlink/fs_test.go @@ -261,3 +261,142 @@ func TestFollowSymlinkRelativePath2(t *testing.T) { t.Fatal(err) } } + +func TestFollowSymlinkScopeLink(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkScopeLink") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + if err := makeFs(tmpdir, []dirOrLink{ + {path: "root2"}, + {path: "root", target: "root2"}, + {path: "root2/foo", target: "../bar"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "root/foo", "root/bar", "root"); err != nil { + t.Fatal(err) + } +} + +func TestFollowSymlinkRootScope(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRootScope") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + expected, err := filepath.EvalSymlinks(tmpdir) + if err != nil { + t.Fatal(err) + } + rewrite, err := FollowSymlinkInScope(tmpdir, "/") + if err != nil { + t.Fatal(err) + } + if rewrite != expected { + t.Fatalf("expected %q got %q", expected, rewrite) + } +} + +func TestFollowSymlinkEmpty(t *testing.T) { + res, err := FollowSymlinkInScope("", "") + if err != nil { + t.Fatal(err) + } + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if res != wd { + t.Fatal("expected %q got %q", wd, res) + } +} + +func TestFollowSymlinkCircular(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkCircular") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + if err := makeFs(tmpdir, []dirOrLink{{path: "root/foo", target: "foo"}}); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "root/foo", "", "root"); err == nil { + t.Fatal("expected an error for foo -> foo") + } + + if err := makeFs(tmpdir, []dirOrLink{ + {path: "root/bar", target: "baz"}, + {path: "root/baz", target: "../bak"}, + {path: "root/bak", target: "/bar"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "root/foo", "", "root"); err == nil { + t.Fatal("expected an error for bar -> baz -> bak -> bar") + } +} + +func TestFollowSymlinkComplexChainWithTargetPathsContainingLinks(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkComplexChainWithTargetPathsContainingLinks") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + if err := makeFs(tmpdir, []dirOrLink{ + {path: "root2"}, + {path: "root", target: "root2"}, + {path: "root/a", target: "r/s"}, + {path: "root/r", target: "../root/t"}, + {path: "root/root/t/s/b", target: "/../u"}, + {path: "root/u/c", target: "."}, + {path: "root/u/x/y", target: "../v"}, + {path: "root/u/v", target: "/../w"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "root/a/b/c/x/y/z", "root/w/z", "root"); err != nil { + t.Fatal(err) + } +} + +func TestFollowSymlinkBreakoutNonExistent(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkBreakoutNonExistent") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + if err := makeFs(tmpdir, []dirOrLink{ + {path: "root/slash", target: "/"}, + {path: "root/sym", target: "/idontexist/../slash"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "root/sym/file", "root/file", "root"); err != nil { + t.Fatal(err) + } +} + +func TestFollowSymlinkNoLexicalCleaning(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkNoLexicalCleaning") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + if err := makeFs(tmpdir, []dirOrLink{ + {path: "root/sym", target: "/foo/bar"}, + {path: "root/hello", target: "/sym/../baz"}, + }); err != nil { + t.Fatal(err) + } + if err := testSymlink(tmpdir, "root/hello", "root/foo/baz", "root"); err != nil { + t.Fatal(err) + } +} From 1cd89729d59948a4bdc9d6c8a4ab01cedaeb193f Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 3 Dec 2014 14:04:51 -0500 Subject: [PATCH 50/67] Add another symlink breakout test Signed-off-by: Tibor Vass --- integration-cli/docker_cli_build_test.go | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 0885f9131..7d557b136 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -3175,3 +3175,57 @@ func TestBuildExoticShellInterpolation(t *testing.T) { logDone("build - exotic shell interpolation") } + +func TestBuildSymlinkBreakout(t *testing.T) { + name := "testbuildsymlinkbreakout" + tmpdir, err := ioutil.TempDir("", name) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + ctx := filepath.Join(tmpdir, "context") + if err := os.MkdirAll(ctx, 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(ctx, "Dockerfile"), []byte(` + from busybox + add symlink.tar / + add inject /symlink/ + `), 0644); err != nil { + t.Fatal(err) + } + inject := filepath.Join(ctx, "inject") + if err := ioutil.WriteFile(inject, nil, 0644); err != nil { + t.Fatal(err) + } + f, err := os.Create(filepath.Join(ctx, "symlink.tar")) + if err != nil { + t.Fatal(err) + } + w := tar.NewWriter(f) + w.WriteHeader(&tar.Header{ + Name: "symlink2", + Typeflag: tar.TypeSymlink, + Linkname: "/../../../../../../../../../../../../../../", + Uid: os.Getuid(), + Gid: os.Getgid(), + }) + w.WriteHeader(&tar.Header{ + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: filepath.Join("symlink2", tmpdir), + Uid: os.Getuid(), + Gid: os.Getgid(), + }) + w.Close() + f.Close() + if _, err := buildImageFromContext(name, &FakeContext{Dir: ctx}, false); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(filepath.Join(tmpdir, "inject")); err == nil { + t.Fatal("symlink breakout - inject") + } else if !os.IsNotExist(err) { + t.Fatalf("unexpected error: %v", err) + } + logDone("build - symlink breakout") +} From 134f8e6b474d07e12712e0f2d55b901246e6482e Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 27 Nov 2014 22:14:15 +0200 Subject: [PATCH 51/67] integ-cli: add build test for absolute symlink Signed-off-by: Cristian Staretu --- integration-cli/docker_cli_build_test.go | 75 ++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 7d557b136..0f14fdf54 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -1167,6 +1167,81 @@ func TestBuildCopyDisallowRemote(t *testing.T) { logDone("build - copy - disallow copy from remote") } +func TestBuildAddBadLinks(t *testing.T) { + const ( + dockerfile = ` + FROM scratch + ADD links.tar / + ADD foo.txt /symlink/ + ` + targetFile = "foo.txt" + ) + var ( + name = "test-link-absolute" + ) + defer deleteImages(name) + ctx, err := fakeContext(dockerfile, nil) + if err != nil { + t.Fatal(err) + } + defer ctx.Close() + + tempDir, err := ioutil.TempDir("", "test-link-absolute-temp-") + if err != nil { + t.Fatalf("failed to create temporary directory: %s", tempDir) + } + defer os.RemoveAll(tempDir) + + symlinkTarget := fmt.Sprintf("/../../../../../../../../../../../..%s", tempDir) + tarPath := filepath.Join(ctx.Dir, "links.tar") + nonExistingFile := filepath.Join(tempDir, targetFile) + fooPath := filepath.Join(ctx.Dir, targetFile) + + tarOut, err := os.Create(tarPath) + if err != nil { + t.Fatal(err) + } + + tarWriter := tar.NewWriter(tarOut) + + header := &tar.Header{ + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: symlinkTarget, + Mode: 0755, + Uid: 0, + Gid: 0, + } + + err = tarWriter.WriteHeader(header) + if err != nil { + t.Fatal(err) + } + + tarWriter.Close() + tarOut.Close() + + foo, err := os.Create(fooPath) + if err != nil { + t.Fatal(err) + } + defer foo.Close() + + if _, err := foo.WriteString("test"); err != nil { + t.Fatal(err) + } + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(nonExistingFile); err == nil || err != nil && !os.IsNotExist(err) { + t.Fatalf("%s shouldn't have been written and it shouldn't exist", nonExistingFile) + } + + logDone("build - ADD must add files in container") +} + // Issue #5270 - ensure we throw a better error than "unexpected EOF" // when we can't access files in the context. func TestBuildWithInaccessibleFilesInContext(t *testing.T) { From e629e255d89759ade9b46fecf413c2ba6765f7dd Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 27 Nov 2014 22:39:43 +0200 Subject: [PATCH 52/67] integ-cli: add test for links in volumes Signed-off-by: Cristian Staretu --- integration-cli/docker_cli_build_test.go | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 0f14fdf54..cb05fa87a 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -1242,6 +1242,58 @@ func TestBuildAddBadLinks(t *testing.T) { logDone("build - ADD must add files in container") } +func TestBuildAddBadLinksVolume(t *testing.T) { + const ( + dockerfileTemplate = ` + FROM busybox + RUN ln -s /../../../../../../../../%s /x + VOLUME /x + ADD foo.txt /x/` + targetFile = "foo.txt" + ) + var ( + name = "test-link-absolute-volume" + dockerfile = "" + ) + defer deleteImages(name) + + tempDir, err := ioutil.TempDir("", "test-link-absolute-volume-temp-") + if err != nil { + t.Fatalf("failed to create temporary directory: %s", tempDir) + } + defer os.RemoveAll(tempDir) + + dockerfile = fmt.Sprintf(dockerfileTemplate, tempDir) + nonExistingFile := filepath.Join(tempDir, targetFile) + + ctx, err := fakeContext(dockerfile, nil) + if err != nil { + t.Fatal(err) + } + defer ctx.Close() + fooPath := filepath.Join(ctx.Dir, targetFile) + + foo, err := os.Create(fooPath) + if err != nil { + t.Fatal(err) + } + defer foo.Close() + + if _, err := foo.WriteString("test"); err != nil { + t.Fatal(err) + } + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(nonExistingFile); err == nil || err != nil && !os.IsNotExist(err) { + t.Fatalf("%s shouldn't have been written and it shouldn't exist", nonExistingFile) + } + + logDone("build - ADD should add files in volume") +} + // Issue #5270 - ensure we throw a better error than "unexpected EOF" // when we can't access files in the context. func TestBuildWithInaccessibleFilesInContext(t *testing.T) { From aef842e7dfb534aba22c3c911de525ce9ac12b72 Mon Sep 17 00:00:00 2001 From: Cristian Staretu Date: Mon, 8 Dec 2014 15:40:27 -0500 Subject: [PATCH 53/67] Add build tests covering extraction in chroot Signed-off-by: Cristian Staretu --- integration-cli/docker_cli_build_test.go | 112 ++++++++++++++++ integration-cli/docker_cli_save_load_test.go | 134 +++++++++++++++++++ 2 files changed, 246 insertions(+) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index cb05fa87a..b70f4eb07 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -2994,6 +2994,118 @@ RUN cat /existing-directory-trailing-slash/test/foo | grep Hi` logDone("build - ADD tar") } +func TestBuildAddTarXz(t *testing.T) { + name := "testbuildaddtarxz" + defer deleteImages(name) + + ctx := func() *FakeContext { + dockerfile := ` + FROM busybox + ADD test.tar.xz / + RUN cat /test/foo | grep Hi` + tmpDir, err := ioutil.TempDir("", "fake-context") + testTar, err := os.Create(filepath.Join(tmpDir, "test.tar")) + if err != nil { + t.Fatalf("failed to create test.tar archive: %v", err) + } + defer testTar.Close() + + tw := tar.NewWriter(testTar) + + if err := tw.WriteHeader(&tar.Header{ + Name: "test/foo", + Size: 2, + }); err != nil { + t.Fatalf("failed to write tar file header: %v", err) + } + if _, err := tw.Write([]byte("Hi")); err != nil { + t.Fatalf("failed to write tar file content: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("failed to close tar archive: %v", err) + } + xzCompressCmd := exec.Command("xz", "test.tar") + xzCompressCmd.Dir = tmpDir + out, _, err := runCommandWithOutput(xzCompressCmd) + if err != nil { + t.Fatal(err, out) + } + + if err := ioutil.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0644); err != nil { + t.Fatalf("failed to open destination dockerfile: %v", err) + } + return &FakeContext{Dir: tmpDir} + }() + + defer ctx.Close() + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + t.Fatalf("build failed to complete for TestBuildAddTarXz: %v", err) + } + + logDone("build - ADD tar.xz") +} + +func TestBuildAddTarXzGz(t *testing.T) { + name := "testbuildaddtarxzgz" + defer deleteImages(name) + + ctx := func() *FakeContext { + dockerfile := ` + FROM busybox + ADD test.tar.xz.gz / + RUN ls /test.tar.xz.gz` + tmpDir, err := ioutil.TempDir("", "fake-context") + testTar, err := os.Create(filepath.Join(tmpDir, "test.tar")) + if err != nil { + t.Fatalf("failed to create test.tar archive: %v", err) + } + defer testTar.Close() + + tw := tar.NewWriter(testTar) + + if err := tw.WriteHeader(&tar.Header{ + Name: "test/foo", + Size: 2, + }); err != nil { + t.Fatalf("failed to write tar file header: %v", err) + } + if _, err := tw.Write([]byte("Hi")); err != nil { + t.Fatalf("failed to write tar file content: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("failed to close tar archive: %v", err) + } + + xzCompressCmd := exec.Command("xz", "test.tar") + xzCompressCmd.Dir = tmpDir + out, _, err := runCommandWithOutput(xzCompressCmd) + if err != nil { + t.Fatal(err, out) + } + + gzipCompressCmd := exec.Command("gzip", "test.tar.xz") + gzipCompressCmd.Dir = tmpDir + out, _, err = runCommandWithOutput(gzipCompressCmd) + if err != nil { + t.Fatal(err, out) + } + + if err := ioutil.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0644); err != nil { + t.Fatalf("failed to open destination dockerfile: %v", err) + } + return &FakeContext{Dir: tmpDir} + }() + + defer ctx.Close() + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + t.Fatalf("build failed to complete for TestBuildAddTarXz: %v", err) + } + + logDone("build - ADD tar.xz.gz") +} + func TestBuildFromGIT(t *testing.T) { name := "testbuildfromgit" defer deleteImages(name) diff --git a/integration-cli/docker_cli_save_load_test.go b/integration-cli/docker_cli_save_load_test.go index 8632247b7..f15e6b51d 100644 --- a/integration-cli/docker_cli_save_load_test.go +++ b/integration-cli/docker_cli_save_load_test.go @@ -62,6 +62,140 @@ func TestSaveAndLoadRepoStdout(t *testing.T) { logDone("load - load a repo using stdout") } +// save a repo using gz compression and try to load it using stdout +func TestSaveXzAndLoadRepoStdout(t *testing.T) { + tempDir, err := ioutil.TempDir("", "test-save-xz-gz-load-repo-stdout") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + tarballPath := filepath.Join(tempDir, "foobar-save-load-test.tar.xz.gz") + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + t.Fatalf("failed to create a container: %v %v", out, err) + } + + cleanedContainerID := stripTrailingCharacters(out) + + repoName := "foobar-save-load-test-xz-gz" + + inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID) + out, _, err = runCommandWithOutput(inspectCmd) + if err != nil { + t.Fatalf("output should've been a container id: %v %v", cleanedContainerID, err) + } + + commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName) + out, _, err = runCommandWithOutput(commitCmd) + if err != nil { + t.Fatalf("failed to commit container: %v %v", out, err) + } + + inspectCmd = exec.Command(dockerBinary, "inspect", repoName) + before, _, err := runCommandWithOutput(inspectCmd) + if err != nil { + t.Fatalf("the repo should exist before saving it: %v %v", before, err) + } + + saveCmdTemplate := `%v save %v | xz -c | gzip -c > %s` + saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName, tarballPath) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + if err != nil { + t.Fatalf("failed to save repo: %v %v", out, err) + } + + deleteImages(repoName) + + loadCmdFinal := fmt.Sprintf(`cat %s | docker load`, tarballPath) + loadCmd := exec.Command("bash", "-c", loadCmdFinal) + out, _, err = runCommandWithOutput(loadCmd) + if err == nil { + t.Fatalf("expected error, but succeeded with no error and output: %v", out) + } + + inspectCmd = exec.Command(dockerBinary, "inspect", repoName) + after, _, err := runCommandWithOutput(inspectCmd) + if err == nil { + t.Fatalf("the repo should not exist: %v", after) + } + + deleteContainer(cleanedContainerID) + deleteImages(repoName) + + logDone("load - save a repo with xz compression & load it using stdout") +} + +// save a repo using xz+gz compression and try to load it using stdout +func TestSaveXzGzAndLoadRepoStdout(t *testing.T) { + tempDir, err := ioutil.TempDir("", "test-save-xz-gz-load-repo-stdout") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + tarballPath := filepath.Join(tempDir, "foobar-save-load-test.tar.xz.gz") + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + t.Fatalf("failed to create a container: %v %v", out, err) + } + + cleanedContainerID := stripTrailingCharacters(out) + + repoName := "foobar-save-load-test-xz-gz" + + inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID) + out, _, err = runCommandWithOutput(inspectCmd) + if err != nil { + t.Fatalf("output should've been a container id: %v %v", cleanedContainerID, err) + } + + commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName) + out, _, err = runCommandWithOutput(commitCmd) + if err != nil { + t.Fatalf("failed to commit container: %v %v", out, err) + } + + inspectCmd = exec.Command(dockerBinary, "inspect", repoName) + before, _, err := runCommandWithOutput(inspectCmd) + if err != nil { + t.Fatalf("the repo should exist before saving it: %v %v", before, err) + } + + saveCmdTemplate := `%v save %v | xz -c | gzip -c > %s` + saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName, tarballPath) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + if err != nil { + t.Fatalf("failed to save repo: %v %v", out, err) + } + + deleteImages(repoName) + + loadCmdFinal := fmt.Sprintf(`cat %s | docker load`, tarballPath) + loadCmd := exec.Command("bash", "-c", loadCmdFinal) + out, _, err = runCommandWithOutput(loadCmd) + if err == nil { + t.Fatalf("expected error, but succeeded with no error and output: %v", out) + } + + inspectCmd = exec.Command(dockerBinary, "inspect", repoName) + after, _, err := runCommandWithOutput(inspectCmd) + if err == nil { + t.Fatalf("the repo should not exist: %v", after) + } + + deleteContainer(cleanedContainerID) + deleteImages(repoName) + + logDone("load - save a repo with xz+gz compression & load it using stdout") +} + func TestSaveSingleTag(t *testing.T) { repoName := "foobar-save-single-tag-test" From 62d83404b5f4c9ecdc21e8cab2306a933ce0cdbc Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 8 Dec 2014 16:14:56 -0500 Subject: [PATCH 54/67] Update chroot apply layer to handle decompression outside chroot Signed-off-by: Michael Crosby Conflicts: pkg/archive/diff.go pkg/chrootarchive/archive.go --- pkg/archive/diff.go | 31 ++++++++++++++++--------------- pkg/chrootarchive/archive.go | 25 ++++++++++++++++++++----- pkg/chrootarchive/diff.go | 28 ++++++++++++++++++++-------- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index 5ed1a1d90..e5467637b 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -21,20 +21,7 @@ func mkdev(major int64, minor int64) uint32 { return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff)) } -// ApplyLayer parses a diff in the standard layer format from `layer`, and -// applies it to the directory `dest`. -func ApplyLayer(dest string, layer ArchiveReader) error { - dest = filepath.Clean(dest) - - // We need to be able to set any perms - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - layer, err := DecompressStream(layer) - if err != nil { - return err - } - +func UnpackLayer(dest string, layer ArchiveReader) error { tr := tar.NewReader(layer) trBuf := pools.BufioReader32KPool.Get(tr) defer pools.BufioReader32KPool.Put(trBuf) @@ -159,6 +146,20 @@ func ApplyLayer(dest string, layer ArchiveReader) error { return err } } - return nil } + +// ApplyLayer parses a diff in the standard layer format from `layer`, and +// applies it to the directory `dest`. +func ApplyLayer(dest string, layer ArchiveReader) error { + dest = filepath.Clean(dest) + // We need to be able to set any perms + oldmask := syscall.Umask(0) + defer syscall.Umask(oldmask) + + layer, err := DecompressStream(layer) + if err != nil { + return err + } + return UnpackLayer(dest, layer) +} diff --git a/pkg/chrootarchive/archive.go b/pkg/chrootarchive/archive.go index fc2bea2c4..7834fb34f 100644 --- a/pkg/chrootarchive/archive.go +++ b/pkg/chrootarchive/archive.go @@ -15,6 +15,15 @@ import ( "github.com/docker/docker/pkg/reexec" ) +var chrootArchiver = &archive.Archiver{Untar} + +func chroot(path string) error { + if err := syscall.Chroot(path); err != nil { + return err + } + return syscall.Chdir("/") +} + func untar() { runtime.LockOSThread() flag.Parse() @@ -36,11 +45,17 @@ func untar() { os.Exit(0) } -var ( - chrootArchiver = &archive.Archiver{Untar} -) +func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { + if tarArchive == nil { + return fmt.Errorf("Empty archive") + } + if options == nil { + options = &archive.TarOptions{} + } + if options.Excludes == nil { + options.Excludes = []string{} + } -func Untar(archive io.Reader, dest string, options *archive.TarOptions) error { var buf bytes.Buffer enc := json.NewEncoder(&buf) if err := enc.Encode(options); err != nil { @@ -53,7 +68,7 @@ func Untar(archive io.Reader, dest string, options *archive.TarOptions) error { } cmd := reexec.Command("docker-untar", dest, buf.String()) - cmd.Stdin = archive + cmd.Stdin = tarArchive out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("Untar %s %s", err, out) diff --git a/pkg/chrootarchive/diff.go b/pkg/chrootarchive/diff.go index 2653aefe9..0fdb3b20e 100644 --- a/pkg/chrootarchive/diff.go +++ b/pkg/chrootarchive/diff.go @@ -3,8 +3,10 @@ package chrootarchive import ( "flag" "fmt" + "io" "io/ioutil" "os" + "path/filepath" "runtime" "syscall" @@ -16,28 +18,38 @@ func applyLayer() { runtime.LockOSThread() flag.Parse() - if err := syscall.Chroot(flag.Arg(0)); err != nil { - fatal(err) - } - if err := syscall.Chdir("/"); err != nil { + if err := chroot(flag.Arg(0)); err != nil { fatal(err) } + // We need to be able to set any perms + oldmask := syscall.Umask(0) + defer syscall.Umask(oldmask) tmpDir, err := ioutil.TempDir("/", "temp-docker-extract") if err != nil { fatal(err) } os.Setenv("TMPDIR", tmpDir) - if err := archive.ApplyLayer("/", os.Stdin); err != nil { - os.RemoveAll(tmpDir) + err = archive.UnpackLayer("/", os.Stdin) + os.RemoveAll(tmpDir) + if err != nil { fatal(err) } - os.RemoveAll(tmpDir) os.Exit(0) } func ApplyLayer(dest string, layer archive.ArchiveReader) error { + dest = filepath.Clean(dest) + decompressed, err := archive.DecompressStream(layer) + if err != nil { + return err + } + defer func() { + if c, ok := decompressed.(io.Closer); ok { + c.Close() + } + }() cmd := reexec.Command("docker-applyLayer", dest) - cmd.Stdin = layer + cmd.Stdin = decompressed out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("ApplyLayer %s %s", err, out) From 313a1b7620910e47d888f8b0a6a5eb06ad9c1ff2 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 8 Dec 2014 16:19:24 -0500 Subject: [PATCH 55/67] Decompress archive before streaming the unpack in a chroot Signed-off-by: Michael Crosby Conflicts: pkg/archive/archive.go pkg/chrootarchive/archive.go --- pkg/archive/archive.go | 52 +++++++++++++++++------------------- pkg/chrootarchive/archive.go | 28 ++++++++++--------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 155145f5d..13f447ca4 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -432,32 +432,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) return pipeReader, nil } -// Untar reads a stream of bytes from `archive`, parses it as a tar archive, -// and unpacks it into the directory at `path`. -// The archive may be compressed with one of the following algorithms: -// identity (uncompressed), gzip, bzip2, xz. -// FIXME: specify behavior when target path exists vs. doesn't exist. -func Untar(archive io.Reader, dest string, options *TarOptions) error { - dest = filepath.Clean(dest) - - if options == nil { - options = &TarOptions{} - } - - if archive == nil { - return fmt.Errorf("Empty archive") - } - - if options.Excludes == nil { - options.Excludes = []string{} - } - - decompressedArchive, err := DecompressStream(archive) - if err != nil { - return err - } - defer decompressedArchive.Close() - +func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { tr := tar.NewReader(decompressedArchive) trBuf := pools.BufioReader32KPool.Get(nil) defer pools.BufioReader32KPool.Put(trBuf) @@ -537,10 +512,33 @@ loop: return err } } - return nil } +// Untar reads a stream of bytes from `archive`, parses it as a tar archive, +// and unpacks it into the directory at `dest`. +// The archive may be compressed with one of the following algorithms: +// identity (uncompressed), gzip, bzip2, xz. +// FIXME: specify behavior when target path exists vs. doesn't exist. +func Untar(archive io.Reader, dest string, options *TarOptions) error { + if archive == nil { + return fmt.Errorf("Empty archive") + } + dest = filepath.Clean(dest) + if options == nil { + options = &TarOptions{} + } + if options.Excludes == nil { + options.Excludes = []string{} + } + decompressedArchive, err := DecompressStream(archive) + if err != nil { + return err + } + defer decompressedArchive.Close() + return Unpack(decompressedArchive, dest, options) +} + func (archiver *Archiver) TarUntar(src, dst string) error { log.Debugf("TarUntar(%s %s)", src, dst) archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed}) diff --git a/pkg/chrootarchive/archive.go b/pkg/chrootarchive/archive.go index 7834fb34f..41eb79e1b 100644 --- a/pkg/chrootarchive/archive.go +++ b/pkg/chrootarchive/archive.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "path/filepath" "runtime" "strings" "syscall" @@ -27,19 +28,14 @@ func chroot(path string) error { func untar() { runtime.LockOSThread() flag.Parse() - - if err := syscall.Chroot(flag.Arg(0)); err != nil { + if err := chroot(flag.Arg(0)); err != nil { fatal(err) } - if err := syscall.Chdir("/"); err != nil { + var options *archive.TarOptions + if err := json.NewDecoder(strings.NewReader(flag.Arg(1))).Decode(&options); err != nil { fatal(err) } - options := new(archive.TarOptions) - dec := json.NewDecoder(strings.NewReader(flag.Arg(1))) - if err := dec.Decode(options); err != nil { - fatal(err) - } - if err := archive.Untar(os.Stdin, "/", options); err != nil { + if err := archive.Unpack(os.Stdin, "/", options); err != nil { fatal(err) } os.Exit(0) @@ -56,8 +52,10 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error options.Excludes = []string{} } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) + var ( + buf bytes.Buffer + enc = json.NewEncoder(&buf) + ) if err := enc.Encode(options); err != nil { return fmt.Errorf("Untar json encode: %v", err) } @@ -66,9 +64,15 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error return err } } + dest = filepath.Clean(dest) + decompressedArchive, err := archive.DecompressStream(tarArchive) + if err != nil { + return err + } + defer decompressedArchive.Close() cmd := reexec.Command("docker-untar", dest, buf.String()) - cmd.Stdin = tarArchive + cmd.Stdin = decompressedArchive out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("Untar %s %s", err, out) From 0e9a7bc3cea20db258176b6f923c2b9cd009334f Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Mon, 8 Dec 2014 14:33:46 -0800 Subject: [PATCH 56/67] Add integration test for xz path issue Signed-off-by: Arnaud Porterie Conflicts: integration-cli/docker_cli_build_test.go --- integration-cli/docker_cli_build_test.go | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index b70f4eb07..408b80161 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -3468,3 +3468,32 @@ func TestBuildSymlinkBreakout(t *testing.T) { } logDone("build - symlink breakout") } + +func TestBuildXZHost(t *testing.T) { + name := "testbuildxzhost" + defer deleteImages(name) + + ctx, err := fakeContext(` +FROM busybox +ADD xz /usr/local/sbin/ +RUN chmod 755 /usr/local/sbin/xz +ADD test.xz / +RUN [ ! -e /injected ]`, + map[string]string{ + "test.xz": "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00" + + "\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x3f\xfd" + + "\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21", + "xz": "#!/bin/sh\ntouch /injected", + }) + + if err != nil { + t.Fatal(err) + } + defer ctx.Close() + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + t.Fatal(err) + } + + logDone("build - xz host is being used") +} From acf1720b3f6035f0ab2c9658371ae8302851ffa3 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 27 Nov 2014 23:55:03 +0200 Subject: [PATCH 57/67] validate image ID properly & before load Signed-off-by: Cristian Staretu --- graph/load.go | 5 +++++ graph/tags_unit_test.go | 2 +- registry/registry.go | 4 ++-- utils/utils.go | 12 +++++++----- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/graph/load.go b/graph/load.go index fcbeef619..f27aca4a6 100644 --- a/graph/load.go +++ b/graph/load.go @@ -12,6 +12,7 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/log" + "github.com/docker/docker/utils" ) // Loads a set of images into the repository. This is the complementary of ImageExport. @@ -112,6 +113,10 @@ func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string log.Debugf("Error unmarshalling json", err) return err } + if err := utils.ValidateID(img.ID); err != nil { + log.Debugf("Error validating ID: %s", err) + return err + } if img.Parent != "" { if !s.graph.Exists(img.Parent) { if err := s.recursiveLoad(eng, img.Parent, tmpImageDir); err != nil { diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index da512547d..bf94deb44 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -16,7 +16,7 @@ import ( const ( testImageName = "myapp" - testImageID = "foo" + testImageID = "1a2d3c4d4e5fa2d2a21acea242a5e2345d3aefc3e7dfa2a2a2a21a2a2ad2d234" ) func fakeTar() (io.Reader, error) { diff --git a/registry/registry.go b/registry/registry.go index a03790af0..e0285a233 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -23,7 +23,6 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") - validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) @@ -177,7 +176,8 @@ func validateRepositoryName(repositoryName string) error { namespace = "library" name = nameParts[0] - if validHex.MatchString(name) { + // the repository name must not be a valid image ID + if err := utils.ValidateID(name); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) } } else { diff --git a/utils/utils.go b/utils/utils.go index 792b80bd5..4c65f136a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -31,6 +31,10 @@ type KeyValuePair struct { Value string } +var ( + validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) +) + // Request a given URL and return an io.Reader func Download(url string) (resp *http.Response, err error) { if resp, err = http.Get(url); err != nil { @@ -190,11 +194,9 @@ func GenerateRandomID() string { } func ValidateID(id string) error { - if id == "" { - return fmt.Errorf("Id can't be empty") - } - if strings.Contains(id, ":") { - return fmt.Errorf("Invalid character in id: ':'") + if ok := validHex.MatchString(id); !ok { + err := fmt.Errorf("image ID '%s' is invalid", id) + return err } return nil } From f2008c535943669a4db89e3b9927a75121effbd5 Mon Sep 17 00:00:00 2001 From: Lewis Marshall Date: Sat, 6 Dec 2014 02:30:03 +0000 Subject: [PATCH 58/67] Fix chroot untar for zero padded archive from slow reader Signed-off-by: Lewis Marshall --- pkg/chrootarchive/archive.go | 5 ++++ pkg/chrootarchive/archive_test.go | 41 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/pkg/chrootarchive/archive.go b/pkg/chrootarchive/archive.go index 41eb79e1b..f9108c534 100644 --- a/pkg/chrootarchive/archive.go +++ b/pkg/chrootarchive/archive.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "os" "path/filepath" "runtime" @@ -38,6 +39,10 @@ func untar() { if err := archive.Unpack(os.Stdin, "/", options); err != nil { fatal(err) } + // fully consume stdin in case it is zero padded + if _, err := ioutil.ReadAll(os.Stdin); err != nil { + fatal(err) + } os.Exit(0) } diff --git a/pkg/chrootarchive/archive_test.go b/pkg/chrootarchive/archive_test.go index 69e18e319..8477c0676 100644 --- a/pkg/chrootarchive/archive_test.go +++ b/pkg/chrootarchive/archive_test.go @@ -1,10 +1,12 @@ package chrootarchive import ( + "io" "io/ioutil" "os" "path/filepath" "testing" + "time" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/reexec" @@ -42,3 +44,42 @@ func TestChrootTarUntar(t *testing.T) { t.Fatal(err) } } + +type slowEmptyTarReader struct { + size int + offset int + chunkSize int +} + +// Read is a slow reader of an empty tar (like the output of "tar c --files-from /dev/null") +func (s *slowEmptyTarReader) Read(p []byte) (int, error) { + time.Sleep(100 * time.Millisecond) + count := s.chunkSize + if len(p) < s.chunkSize { + count = len(p) + } + for i := 0; i < count; i++ { + p[i] = 0 + } + s.offset += count + if s.offset > s.size { + return count, io.EOF + } + return count, nil +} + +func TestChrootUntarEmptyArchiveFromSlowReader(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarEmptyArchiveFromSlowReader") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + dest := filepath.Join(tmpdir, "dest") + if err := os.MkdirAll(dest, 0700); err != nil { + t.Fatal(err) + } + stream := &slowEmptyTarReader{size: 10240, chunkSize: 1024} + if err := Untar(stream, dest, nil); err != nil { + t.Fatal(err) + } +} From c8fc8768b672b06bd0cf38d65da8ba03016b85b2 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 8 Dec 2014 15:04:34 -0800 Subject: [PATCH 59/67] Flush stdin from within chroot archive This makes sure that we don't buffer in memory and that we also flush stdin from diff as well as untar. Signed-off-by: Michael Crosby Conflicts: pkg/chrootarchive/diff.go --- pkg/chrootarchive/archive.go | 5 +---- pkg/chrootarchive/archive_test.go | 16 ++++++++++++++++ pkg/chrootarchive/diff.go | 2 ++ pkg/chrootarchive/init.go | 8 ++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/chrootarchive/archive.go b/pkg/chrootarchive/archive.go index f9108c534..0077f930d 100644 --- a/pkg/chrootarchive/archive.go +++ b/pkg/chrootarchive/archive.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -40,9 +39,7 @@ func untar() { fatal(err) } // fully consume stdin in case it is zero padded - if _, err := ioutil.ReadAll(os.Stdin); err != nil { - fatal(err) - } + flush(os.Stdin) os.Exit(0) } diff --git a/pkg/chrootarchive/archive_test.go b/pkg/chrootarchive/archive_test.go index 8477c0676..0fe3d64f9 100644 --- a/pkg/chrootarchive/archive_test.go +++ b/pkg/chrootarchive/archive_test.go @@ -83,3 +83,19 @@ func TestChrootUntarEmptyArchiveFromSlowReader(t *testing.T) { t.Fatal(err) } } + +func TestChrootApplyEmptyArchiveFromSlowReader(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "docker-TestChrootApplyEmptyArchiveFromSlowReader") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + dest := filepath.Join(tmpdir, "dest") + if err := os.MkdirAll(dest, 0700); err != nil { + t.Fatal(err) + } + stream := &slowEmptyTarReader{size: 10240, chunkSize: 1024} + if err := ApplyLayer(dest, stream); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/chrootarchive/diff.go b/pkg/chrootarchive/diff.go index 0fdb3b20e..d4e9529b6 100644 --- a/pkg/chrootarchive/diff.go +++ b/pkg/chrootarchive/diff.go @@ -34,6 +34,8 @@ func applyLayer() { if err != nil { fatal(err) } + os.RemoveAll(tmpDir) + flush(os.Stdin) os.Exit(0) } diff --git a/pkg/chrootarchive/init.go b/pkg/chrootarchive/init.go index b548e9fe7..4116026ef 100644 --- a/pkg/chrootarchive/init.go +++ b/pkg/chrootarchive/init.go @@ -2,6 +2,8 @@ package chrootarchive import ( "fmt" + "io" + "io/ioutil" "os" "github.com/docker/docker/pkg/reexec" @@ -16,3 +18,9 @@ func fatal(err error) { fmt.Fprint(os.Stderr, err) os.Exit(1) } + +// flush consumes all the bytes from the reader discarding +// any errors +func flush(r io.Reader) { + io.Copy(ioutil.Discard, r) +} From 0de96a81636e27a6b7cf3a34a62419545d030234 Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Mon, 1 Dec 2014 17:57:53 -0800 Subject: [PATCH 60/67] Fix client-side HTTP hijacking over TLS Properly CloseWrite() the client socket once done with stdin when using TLS connection (this used to rely on an erroneous type assertion). Fixes #8658. Fixes #8642. Signed-off-by: Arnaud Porterie Signed-off-by: Michael Crosby --- api/client/hijack.go | 105 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 7 deletions(-) diff --git a/api/client/hijack.go b/api/client/hijack.go index 00170a4a3..d0b5e93ef 100644 --- a/api/client/hijack.go +++ b/api/client/hijack.go @@ -2,6 +2,7 @@ package client import ( "crypto/tls" + "errors" "fmt" "io" "net" @@ -10,6 +11,7 @@ import ( "os" "runtime" "strings" + "time" "github.com/docker/docker/api" "github.com/docker/docker/dockerversion" @@ -19,9 +21,99 @@ import ( "github.com/docker/docker/pkg/term" ) +type tlsClientCon struct { + *tls.Conn + rawConn net.Conn +} + +func (c *tlsClientCon) CloseWrite() error { + // Go standard tls.Conn doesn't provide the CloseWrite() method so we do it + // on its underlying connection. + if cwc, ok := c.rawConn.(interface { + CloseWrite() error + }); ok { + return cwc.CloseWrite() + } + return nil +} + +func tlsDial(network, addr string, config *tls.Config) (net.Conn, error) { + return tlsDialWithDialer(new(net.Dialer), network, addr, config) +} + +// We need to copy Go's implementation of tls.Dial (pkg/cryptor/tls/tls.go) in +// order to return our custom tlsClientCon struct which holds both the tls.Conn +// object _and_ its underlying raw connection. The rationale for this is that +// we need to be able to close the write end of the connection when attaching, +// which tls.Conn does not provide. +func tlsDialWithDialer(dialer *net.Dialer, network, addr string, config *tls.Config) (net.Conn, error) { + // We want the Timeout and Deadline values from dialer to cover the + // whole process: TCP connection and TLS handshake. This means that we + // also need to start our own timers now. + timeout := dialer.Timeout + + if !dialer.Deadline.IsZero() { + deadlineTimeout := dialer.Deadline.Sub(time.Now()) + if timeout == 0 || deadlineTimeout < timeout { + timeout = deadlineTimeout + } + } + + var errChannel chan error + + if timeout != 0 { + errChannel = make(chan error, 2) + time.AfterFunc(timeout, func() { + errChannel <- errors.New("") + }) + } + + rawConn, err := dialer.Dial(network, addr) + if err != nil { + return nil, err + } + + colonPos := strings.LastIndex(addr, ":") + if colonPos == -1 { + colonPos = len(addr) + } + hostname := addr[:colonPos] + + // If no ServerName is set, infer the ServerName + // from the hostname we're connecting to. + if config.ServerName == "" { + // Make a copy to avoid polluting argument or default. + c := *config + c.ServerName = hostname + config = &c + } + + conn := tls.Client(rawConn, config) + + if timeout == 0 { + err = conn.Handshake() + } else { + go func() { + errChannel <- conn.Handshake() + }() + + err = <-errChannel + } + + if err != nil { + rawConn.Close() + return nil, err + } + + // This is Docker difference with standard's crypto/tls package: returned a + // wrapper which holds both the TLS and raw connections. + return &tlsClientCon{conn, rawConn}, nil +} + func (cli *DockerCli) dial() (net.Conn, error) { if cli.tlsConfig != nil && cli.proto != "unix" { - return tls.Dial(cli.proto, cli.addr, cli.tlsConfig) + // Notice this isn't Go standard's tls.Dial function + return tlsDial(cli.proto, cli.addr, cli.tlsConfig) } return net.Dial(cli.proto, cli.addr) } @@ -109,12 +201,11 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea io.Copy(rwc, in) log.Debugf("[hijack] End of stdin") } - if tcpc, ok := rwc.(*net.TCPConn); ok { - if err := tcpc.CloseWrite(); err != nil { - log.Debugf("Couldn't send EOF: %s", err) - } - } else if unixc, ok := rwc.(*net.UnixConn); ok { - if err := unixc.CloseWrite(); err != nil { + + if conn, ok := rwc.(interface { + CloseWrite() error + }); ok { + if err := conn.CloseWrite(); err != nil { log.Debugf("Couldn't send EOF: %s", err) } } From 994e4a1c69c5f48eeba679437fdd7b7ed7ab0fc5 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Wed, 26 Nov 2014 23:00:13 -0800 Subject: [PATCH 61/67] Change path breakout detection logic in archive package Fixes #9375 Signed-off-by: Alexandr Morozov Conflicts: integration-cli/docker_cli_cp_test.go removed extra test --- integration-cli/docker_cli_cp_test.go | 38 +++++++++++++++++++++++++++ pkg/archive/archive.go | 9 ++++--- pkg/archive/diff.go | 12 +++++---- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/integration-cli/docker_cli_cp_test.go b/integration-cli/docker_cli_cp_test.go index aecc68edb..a5e16bb21 100644 --- a/integration-cli/docker_cli_cp_test.go +++ b/integration-cli/docker_cli_cp_test.go @@ -371,3 +371,41 @@ func TestCpUnprivilegedUser(t *testing.T) { logDone("cp - unprivileged user") } + +func TestCpToDot(t *testing.T) { + out, exitCode, err := dockerCmd(t, "run", "-d", "busybox", "/bin/sh", "-c", "echo lololol > /test") + if err != nil || exitCode != 0 { + t.Fatal("failed to create a container", out, err) + } + + cleanedContainerID := stripTrailingCharacters(out) + defer deleteContainer(cleanedContainerID) + + out, _, err = dockerCmd(t, "wait", cleanedContainerID) + if err != nil || stripTrailingCharacters(out) != "0" { + t.Fatal("failed to set up container", out, err) + } + + tmpdir, err := ioutil.TempDir("", "docker-integration") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + defer os.Chdir(cwd) + if err := os.Chdir(tmpdir); err != nil { + t.Fatal(err) + } + _, _, err = dockerCmd(t, "cp", cleanedContainerID+":/test", ".") + if err != nil { + t.Fatalf("couldn't docker cp to \".\" path: %s", err) + } + content, err := ioutil.ReadFile("./test") + if string(content) != "lololol\n" { + t.Fatal("Wrong content in copied file %q, should be %q", content, "lololol\n") + } + logDone("cp - to dot path") +} diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 13f447ca4..74c601450 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -473,10 +473,13 @@ loop: } } - // Prevent symlink breakout path := filepath.Join(dest, hdr.Name) - if !strings.HasPrefix(path, dest) { - return breakoutError(fmt.Errorf("%q is outside of %q", path, dest)) + rel, err := filepath.Rel(dest, path) + if err != nil { + return err + } + if strings.HasPrefix(rel, "..") { + return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) } // If path exits we almost always just want to remove and replace it diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index e5467637b..80bb19746 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -81,12 +81,14 @@ func UnpackLayer(dest string, layer ArchiveReader) error { } path := filepath.Join(dest, hdr.Name) - base := filepath.Base(path) - - // Prevent symlink breakout - if !strings.HasPrefix(path, dest) { - return breakoutError(fmt.Errorf("%q is outside of %q", path, dest)) + rel, err := filepath.Rel(dest, path) + if err != nil { + return err } + if strings.HasPrefix(rel, "..") { + return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) + } + base := filepath.Base(path) if strings.HasPrefix(base, ".wh.") { originalBase := base[len(".wh."):] From ee8504bc5ace8d3352166f1f2dce91b56fd9ad91 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 10 Dec 2014 18:19:04 -0500 Subject: [PATCH 62/67] docs: Add release notes Signed-off-by: Tibor Vass --- docs/sources/release-notes.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/sources/release-notes.md b/docs/sources/release-notes.md index a19eb71b6..1e744e32a 100644 --- a/docs/sources/release-notes.md +++ b/docs/sources/release-notes.md @@ -4,6 +4,40 @@ understanding, release #Release Notes +##Version 1.3.3 +(2014-12-11) + +This release fixes several security issues. In order to encourage immediate +upgrading, this release also patches some critical bugs. All users are highly +encouraged to upgrade as soon as possible. + +*Security fixes* + +Patches and changes were made to address the following vulnerabilities: + +* CVE-2014-9356: Path traversal during processing of absolute symlinks. +Absolute symlinks were not adequately checked for traversal which created a +vulnerability via image extraction and/or volume mounts. +* CVE-2014-9357: Escalation of privileges during decompression of LZMA (.xz) +archives. Docker 1.3.2 added `chroot` for archive extraction. This created a +vulnerability that could allow malicious images or builds to write files to the +host system and escape containerization, leading to privilege escalation. +* CVE-2014-9358: Path traversal and spoofing opportunities via image +identifiers. Image IDs passed either via `docker load` or registry communications +were not sufficiently validated. This created a vulnerability to path traversal +attacks wherein malicious images or repository spoofing could lead to graph +corruption and manipulation. + +*Runtime fixes* + +* Fixed an issue that cause image archives to be read slowly. + +*Client fixes* + +* Fixed a regression related to STDIN redirection. +* Fixed a regression involving `docker cp` when the current directory is the +destination. + ##Version 1.3.2 (2014-11-24) From 496c2748cfef82fea3f5e1be5fbf845b478420aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hans=20R=C3=B8dtang?= Date: Fri, 14 Nov 2014 06:16:41 +0100 Subject: [PATCH 63/67] Updated cover tool import path. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Hans Rødtang --- Dockerfile | 2 +- builder/parser/testfiles/docker/Dockerfile | 2 +- builder/parser/testfiles/docker/result | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9c58a921..bd943deeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ ENV GOARM 5 RUN cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done' # Grab Go's cover tool for dead-simple code coverage testing -RUN go get code.google.com/p/go.tools/cmd/cover +RUN go get golang.org/x/tools/cmd/cover # TODO replace FPM with some very minimal debhelper stuff RUN gem install --no-rdoc --no-ri fpm --version 1.3.2 diff --git a/builder/parser/testfiles/docker/Dockerfile b/builder/parser/testfiles/docker/Dockerfile index 68f8f0b78..fba1d8b99 100644 --- a/builder/parser/testfiles/docker/Dockerfile +++ b/builder/parser/testfiles/docker/Dockerfile @@ -76,7 +76,7 @@ ENV GOARM 5 RUN cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done' # Grab Go's cover tool for dead-simple code coverage testing -RUN go get code.google.com/p/go.tools/cmd/cover +RUN go get golang.org/x/tools/cmd/cover # TODO replace FPM with some very minimal debhelper stuff RUN gem install --no-rdoc --no-ri fpm --version 1.0.2 diff --git a/builder/parser/testfiles/docker/result b/builder/parser/testfiles/docker/result index 915e2ef17..db74e869b 100644 --- a/builder/parser/testfiles/docker/result +++ b/builder/parser/testfiles/docker/result @@ -11,7 +11,7 @@ (env "DOCKER_CROSSPLATFORMS" "linux/386 linux/arm darwin/amd64 darwin/386 freebsd/amd64 freebsd/386 freebsd/arm") (env "GOARM" "5") (run "cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done'") -(run "go get code.google.com/p/go.tools/cmd/cover") +(run "go get golang.org/x/tools/cmd/cover") (run "gem install --no-rdoc --no-ri fpm --version 1.0.2") (run "git clone -b buildroot-2014.02 https://github.com/jpetazzo/docker-busybox.git /docker-busybox") (run "/bin/echo -e '[default]\\naccess_key=$AWS_ACCESS_KEY\\nsecret_key=$AWS_SECRET_KEY' > /.s3cfg") From d3446258471733fe667f0abdf015ee3ebd2f3485 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 1 Dec 2014 13:00:44 +0200 Subject: [PATCH 64/67] Bump version to v1.3.3 --- CHANGELOG.md | 14 ++++++++++++++ VERSION | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cddc6c90..fdc05c106 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 1.3.3 (2014-12-11) + +#### Security +- Fix path traversal vulnerability in processing of absolute symbolic links (CVE-2014-9356) +- Fix decompression of xz image archives, preventing privilege escalation (CVE-2014-9357) +- Validate image IDs (CVE-2014-9358) + +#### Runtime +- Fix an issue when image archives are being read slowly + +#### Client +- Fix a regression related to stdin redirection +- Fix a regression with `docker cp` when destination is the current directory + ## 1.3.2 (2014-11-20) #### Security diff --git a/VERSION b/VERSION index 1892b9267..31e5c8434 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.2 +1.3.3 From 4595d4fb03093acf87b905bebc5ba4964d7c0707 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Thu, 11 Dec 2014 15:00:44 -0800 Subject: [PATCH 65/67] Bump version to v1.4.0 Docker-DCO-1.1-Signed-off-by: Jessica Frazelle (github: jfrazelle) --- CHANGELOG.md | 14 ++++++++++++++ VERSION | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 346fe4ce2..6db1a2a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 1.4.0 (2014-12-11) + +#### Notable Features since 1.3.0 ++ Set key=value labels to the daemon (displayed in `docker info`), applied with + new `-label` daemon flag ++ Add support for `ENV` in Dockerfile of the form: + `ENV name=value name2=value2...` ++ New Overlayfs Storage Driver ++ `docker info` now returns an `ID` and `Name` field ++ Filter events by event name, container, or image ++ `docker cp` now supports copying from container volumes +- Fixed `docker tag`, so it honors `--force` when overriding a tag for existing + image. + ## 1.3.3 (2014-12-11) #### Security diff --git a/VERSION b/VERSION index 456ea726a..88c5fb891 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.3-dev +1.4.0 From c509835475f4f222400e6a4d2211c59e41329817 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Thu, 11 Dec 2014 17:00:43 -0800 Subject: [PATCH 66/67] Change version to 1.4.0-dev Docker-DCO-1.1-Signed-off-by: Jessica Frazelle (github: jfrazelle) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 88c5fb891..c3f0d2bbd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.0 +1.4.0-dev From fd831b4c5e50a3f1de8eb506016ffe89d5cac8d4 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Thu, 11 Dec 2014 17:14:53 -0800 Subject: [PATCH 67/67] Fix conflicts. Docker-DCO-1.1-Signed-off-by: Jessica Frazelle (github: jfrazelle) --- integration-cli/docker_cli_save_load_test.go | 67 ++++++++++++++++++++ registry/registry.go | 4 -- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/integration-cli/docker_cli_save_load_test.go b/integration-cli/docker_cli_save_load_test.go index 6de5a7963..94bfe3d6a 100644 --- a/integration-cli/docker_cli_save_load_test.go +++ b/integration-cli/docker_cli_save_load_test.go @@ -171,6 +171,73 @@ func TestSaveXzAndLoadRepoStdout(t *testing.T) { logDone("load - save a repo with xz compression & load it using stdout") } +// save a repo using xz+gz compression and try to load it using stdout +func TestSaveXzGzAndLoadRepoStdout(t *testing.T) { + tempDir, err := ioutil.TempDir("", "test-save-xz-gz-load-repo-stdout") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + tarballPath := filepath.Join(tempDir, "foobar-save-load-test.tar.xz.gz") + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + t.Fatalf("failed to create a container: %v %v", out, err) + } + + cleanedContainerID := stripTrailingCharacters(out) + + repoName := "foobar-save-load-test-xz-gz" + + inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID) + out, _, err = runCommandWithOutput(inspectCmd) + if err != nil { + t.Fatalf("output should've been a container id: %v %v", cleanedContainerID, err) + } + + commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName) + out, _, err = runCommandWithOutput(commitCmd) + if err != nil { + t.Fatalf("failed to commit container: %v %v", out, err) + } + + inspectCmd = exec.Command(dockerBinary, "inspect", repoName) + before, _, err := runCommandWithOutput(inspectCmd) + if err != nil { + t.Fatalf("the repo should exist before saving it: %v %v", before, err) + } + + saveCmdTemplate := `%v save %v | xz -c | gzip -c > %s` + saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName, tarballPath) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + if err != nil { + t.Fatalf("failed to save repo: %v %v", out, err) + } + + deleteImages(repoName) + + loadCmdFinal := fmt.Sprintf(`cat %s | docker load`, tarballPath) + loadCmd := exec.Command("bash", "-c", loadCmdFinal) + out, _, err = runCommandWithOutput(loadCmd) + if err == nil { + t.Fatalf("expected error, but succeeded with no error and output: %v", out) + } + + inspectCmd = exec.Command(dockerBinary, "inspect", repoName) + after, _, err := runCommandWithOutput(inspectCmd) + if err == nil { + t.Fatalf("the repo should not exist: %v", after) + } + + deleteContainer(cleanedContainerID) + deleteImages(repoName) + + logDone("load - save a repo with xz+gz compression & load it using stdout") +} + func TestSaveSingleTag(t *testing.T) { repoName := "foobar-save-single-tag-test" diff --git a/registry/registry.go b/registry/registry.go index f3a4a340b..d503a63d6 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -47,10 +47,6 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate tlsConfig.InsecureSkipVerify = true } - if !secure { - tlsConfig.InsecureSkipVerify = true - } - httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment,