From 3e8626c4a171653971aebcb778087c89c6c7ab67 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 8 Jul 2013 17:20:41 -0700 Subject: [PATCH 01/10] Changed the tag parsing to it will work even if there is a port in the repos registry url (full qualified name for pushing on a standalone registry) --- buildfile.go | 12 +----------- commands.go | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/buildfile.go b/buildfile.go index 1c5e8289b..411f44bc3 100644 --- a/buildfile.go +++ b/buildfile.go @@ -52,20 +52,10 @@ func (b *buildFile) CmdFrom(name string) error { image, err := b.runtime.repositories.LookupImage(name) if err != nil { if b.runtime.graph.IsNotExist(err) { - - var tag, remote string - if strings.Contains(name, ":") { - remoteParts := strings.Split(name, ":") - tag = remoteParts[1] - remote = remoteParts[0] - } else { - remote = name - } - + tag, remote := parseRepositoryTag(name) if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { return err } - image, err = b.runtime.repositories.LookupImage(name) if err != nil { return err diff --git a/commands.go b/commands.go index 0bd7965f7..3134011c0 100644 --- a/commands.go +++ b/commands.go @@ -754,6 +754,20 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } +// Get a repos name and returns the right reposName + tag +// The tag can be confusing because of a port in a repository name. +// Ex: localhost.localdomain:5000/samalba/hipache:latest +func parseRepositoryTag(repos string) (string, string) { + n := strings.LastIndex(repos, ":") + if n < 0 { + return repos, "" + } + if tag := repos[n+1:]; !strings.Contains(tag, "/") { + return repos[:n], tag + } + return repos, "" +} + func (cli *DockerCli) CmdPull(args ...string) error { cmd := Subcmd("pull", "NAME", "Pull an image or a repository from the registry") tag := cmd.String("t", "", "Download tagged image in repository") @@ -766,12 +780,8 @@ func (cli *DockerCli) CmdPull(args ...string) error { return nil } - remote := cmd.Arg(0) - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") - tag = &remoteParts[1] - remote = remoteParts[0] - } + remote, parsedTag := parseRepositoryTag(cmd.Arg(0)) + *tag = parsedTag v := url.Values{} v.Set("fromImage", remote) From e7d36c9590bfdc2e57cbbecce87729f51ac3b35f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 8 Jul 2013 17:22:41 -0700 Subject: [PATCH 02/10] It is now possible to include a ":" in a local repository name (it will not be the case for a remote name). This adds support for full qualified repository name in order to support private registry server --- tags.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tags.go b/tags.go index d1eb36aa7..f7307ed42 100644 --- a/tags.go +++ b/tags.go @@ -221,9 +221,6 @@ func validateRepoName(name string) error { if name == "" { return fmt.Errorf("Repository name can't be empty") } - if strings.Contains(name, ":") { - return fmt.Errorf("Illegal repository name: %s", name) - } return nil } From 31c66d5a00ee693a31902d1f75777dc9507c1416 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 8 Jul 2013 17:26:50 -0700 Subject: [PATCH 03/10] Re-implemented a notion of local and private repos. This allows to consider the full qualified name of the repos as the name for the local repository without breaking the calls to the Registry API. --- server.go | 73 ++++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/server.go b/server.go index 71bcd29ec..f535da6fc 100644 --- a/server.go +++ b/server.go @@ -351,10 +351,10 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, name, askedTag, indexEp string, sf *utils.StreamFormatter) error { - out.Write(sf.FormatStatus("Pulling repository %s from %s", name, indexEp)) +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error { + out.Write(sf.FormatStatus("Pulling repository %s", localName)) - repoData, err := r.GetRepositoryData(indexEp, name) + repoData, err := r.GetRepositoryData(indexEp, remoteName) if err != nil { return err } @@ -366,7 +366,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, name, ask } utils.Debugf("Retrieving the tag list") - tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) + tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { utils.Debugf("%v", err) return err @@ -390,7 +390,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, name, ask // Otherwise, check that the tag exists and use only that one id, exists := tagsList[askedTag] if !exists { - return fmt.Errorf("Tag %s not found in repository %s", askedTag, name) + return fmt.Errorf("Tag %s not found in repository %s", askedTag, localName) } repoData.ImgList[id].Tag = askedTag } @@ -405,7 +405,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, name, ask utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) continue } - out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, name)) + out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) success := false for _, ep := range repoData.Endpoints { if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { @@ -423,7 +423,7 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, name, ask if askedTag != "" && tag != askedTag { continue } - if err := srv.runtime.repositories.Set(name, tag, id, true); err != nil { + if err := srv.runtime.repositories.Set(localName, tag, id, true); err != nil { return err } } @@ -469,27 +469,26 @@ func (srv *Server) poolRemove(kind, key string) error { return nil } -func (srv *Server) ImagePull(name string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { +func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig) if err != nil { return err } - if err := srv.poolAdd("pull", name+":"+tag); err != nil { + if err := srv.poolAdd("pull", localName+":"+tag); err != nil { return err } - defer srv.poolRemove("pull", name+":"+tag) + defer srv.poolRemove("pull", localName+":"+tag) // Resolve the Repository name from fqn to endpoint + name - var endpoint string - endpoint, name, err = registry.ResolveRepositoryName(name) + endpoint, remoteName, err := registry.ResolveRepositoryName(localName) if err != nil { return err } out = utils.NewWriteFlusher(out) - err = srv.pullRepository(r, out, name, tag, endpoint, sf) + err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf) if err != nil { - if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { + if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil { return err } return nil @@ -564,7 +563,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return imgList, nil } -func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) out.Write(sf.FormatStatus("Processing checksums")) imgList, err := srv.getImageList(localRepo) @@ -572,41 +571,36 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri return err } out.Write(sf.FormatStatus("Sending image list")) - srvName := name - parts := strings.Split(name, "/") - if len(parts) > 2 { - srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) - } var repoData *registry.RepositoryData - repoData, err = r.PushImageJSONIndex(indexEp, name, imgList, false, nil) + repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil) if err != nil { return err } for _, ep := range repoData.Endpoints { - out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo))) + out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { - out.Write(sf.FormatStatus("Image %s already on registry, skipping", name)) + out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) { - fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) + out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } - if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil { + if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err } - out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+srvName+"/tags/"+elem.Tag)) - if err := r.PushRegistryTag(srvName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { + out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) + if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err } } } - if _, err := r.PushImageJSONIndex(indexEp, name, imgList, true, repoData.Endpoints); err != nil { + if _, err := r.PushImageJSONIndex(indexEp, remoteName, imgList, true, repoData.Endpoints); err != nil { return err } @@ -634,7 +628,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { - out.Write(sf.FormatStatus("Image %s already uploaded ; skipping", imgData.ID)) + out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) return nil } return err @@ -674,30 +668,31 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // FIXME: Allow to interupt current push when new push of same image is done. -func (srv *Server) ImagePush(name string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - if err := srv.poolAdd("push", name); err != nil { +func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { + if err := srv.poolAdd("push", localName); err != nil { return err } - defer srv.poolRemove("push", name) + defer srv.poolRemove("push", localName) // Resolve the Repository name from fqn to endpoint + name - endpoint, name, err := registry.ResolveRepositoryName(name) + endpoint, remoteName, err := registry.ResolveRepositoryName(localName) if err != nil { return err } out = utils.NewWriteFlusher(out) - img, err := srv.runtime.graph.Get(name) + img, err := srv.runtime.graph.Get(localName) r, err2 := registry.NewRegistry(srv.runtime.root, authConfig) if err2 != nil { return err2 } if err != nil { - out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) + reposLen := len(srv.runtime.repositories.Repositories[localName]) + out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen)) // If it fails, try to get the repository - if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(r, out, name, localRepo, endpoint, sf); err != nil { + if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists { + if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil { return err } return nil @@ -706,8 +701,8 @@ func (srv *Server) ImagePush(name string, out io.Writer, sf *utils.StreamFormatt } var token []string - out.Write(sf.FormatStatus("The push refers to an image: [%s]", name)) - if err := srv.pushImage(r, out, name, img.ID, endpoint, token, sf); err != nil { + out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) + if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } return nil From 3be7bc38e034c94da7cd6da5f2e5ecffd1832d6b Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Mon, 8 Jul 2013 17:42:18 -0700 Subject: [PATCH 04/10] Fixed typo (thanks unit tests) --- buildfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildfile.go b/buildfile.go index 411f44bc3..febcaca8e 100644 --- a/buildfile.go +++ b/buildfile.go @@ -52,7 +52,7 @@ func (b *buildFile) CmdFrom(name string) error { image, err := b.runtime.repositories.LookupImage(name) if err != nil { if b.runtime.graph.IsNotExist(err) { - tag, remote := parseRepositoryTag(name) + remote, tag := parseRepositoryTag(name) if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { return err } From 019324015b0c64733ae4f39708aa49a08134cb48 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 08:06:10 -0700 Subject: [PATCH 05/10] Moved parseRepositoryTag to the utils package --- buildfile.go | 2 +- commands.go | 16 +--------------- utils/utils.go | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/buildfile.go b/buildfile.go index febcaca8e..6e117b19e 100644 --- a/buildfile.go +++ b/buildfile.go @@ -52,7 +52,7 @@ func (b *buildFile) CmdFrom(name string) error { image, err := b.runtime.repositories.LookupImage(name) if err != nil { if b.runtime.graph.IsNotExist(err) { - remote, tag := parseRepositoryTag(name) + remote, tag := utils.ParseRepositoryTag(name) if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { return err } diff --git a/commands.go b/commands.go index 3134011c0..ec8615995 100644 --- a/commands.go +++ b/commands.go @@ -754,20 +754,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } -// Get a repos name and returns the right reposName + tag -// The tag can be confusing because of a port in a repository name. -// Ex: localhost.localdomain:5000/samalba/hipache:latest -func parseRepositoryTag(repos string) (string, string) { - n := strings.LastIndex(repos, ":") - if n < 0 { - return repos, "" - } - if tag := repos[n+1:]; !strings.Contains(tag, "/") { - return repos[:n], tag - } - return repos, "" -} - func (cli *DockerCli) CmdPull(args ...string) error { cmd := Subcmd("pull", "NAME", "Pull an image or a repository from the registry") tag := cmd.String("t", "", "Download tagged image in repository") @@ -780,7 +766,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { return nil } - remote, parsedTag := parseRepositoryTag(cmd.Arg(0)) + remote, parsedTag := utils.ParseRepositoryTag(cmd.Arg(0)) *tag = parsedTag v := url.Values{} diff --git a/utils/utils.go b/utils/utils.go index eee6685c8..df615844a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -686,3 +686,17 @@ func ParseHost(host string, port int, addr string) string { } return fmt.Sprintf("tcp://%s:%d", host, port) } + +// Get a repos name and returns the right reposName + tag +// The tag can be confusing because of a port in a repository name. +// Ex: localhost.localdomain:5000/samalba/hipache:latest +func ParseRepositoryTag(repos string) (string, string) { + n := strings.LastIndex(repos, ":") + if n < 0 { + return repos, "" + } + if tag := repos[n+1:]; !strings.Contains(tag, "/") { + return repos[:n], tag + } + return repos, "" +} From 33d97e81ebc4d25ecc7366aeef625e51aa69478c Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 08:10:43 -0700 Subject: [PATCH 06/10] Removed DOCKER_INDEX_URL --- auth/auth.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 2e52af88d..97df928b6 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -41,9 +41,6 @@ func NewAuthConfig(username, password, email, rootPath string) *AuthConfig { } func IndexServerAddress() string { - if os.Getenv("DOCKER_INDEX_URL") != "" { - return os.Getenv("DOCKER_INDEX_URL") + "/v1/" - } return INDEXSERVER } From f44eac49fae1b33e6ff4c6f42c5e7305caf22252 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 11:30:12 -0700 Subject: [PATCH 07/10] Fixed potential security issue (never try http on official index when polling the endpoint). Also fixed local repos name when pulling index.docker.io/foo/bar --- registry/registry.go | 14 ++++++++++++-- server.go | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index c458f616f..2f225aed9 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -18,8 +18,14 @@ import ( ) var ErrAlreadyExists = errors.New("Image already exists") +var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") func pingRegistryEndpoint(endpoint string) error { + if endpoint == auth.IndexServerAddress() { + // Skip the check, we now this one is valid + // (and we never want to fallback to http in case of error) + return nil + } resp, err := http.Get(endpoint + "_ping") if err != nil { return err @@ -56,16 +62,20 @@ func validateRepositoryName(repositoryName string) error { // Resolves a repository name to a endpoint + name func ResolveRepositoryName(reposName string) (string, string, error) { + if strings.Contains(reposName, "://") { + // It cannot contain a scheme! + return "", "", ErrInvalidRepositoryName + } nameParts := strings.SplitN(reposName, "/", 2) if !strings.Contains(nameParts[0], ".") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) - return "https://index.docker.io/v1/", reposName, err + return auth.IndexServerAddress(), reposName, err } if len(nameParts) < 2 { // There is a dot in repos name (and no registry address) // Is it a Registry address without repos name? - return "", "", fmt.Errorf("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + return "", "", ErrInvalidRepositoryName } hostname := nameParts[0] reposName = nameParts[1] diff --git a/server.go b/server.go index f535da6fc..f1c090951 100644 --- a/server.go +++ b/server.go @@ -485,6 +485,11 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut return err } + if endpoint == auth.IndexServerAddress() { + // If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar" + localName = remoteName + } + out = utils.NewWriteFlusher(out) err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf) if err != nil { From 59b785a2820e9ff07ce2a71c83eef8b6ccc467cc Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 16:45:32 -0700 Subject: [PATCH 08/10] Fixing missing tag field when pulling containers which does not exist --- commands.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index bce0ba549..feab55825 100644 --- a/commands.go +++ b/commands.go @@ -1242,7 +1242,9 @@ func (cli *DockerCli) CmdRun(args ...string) error { //if image not found try to pull it if statusCode == 404 { v := url.Values{} - v.Set("fromImage", config.Image) + repos, tag := utils.ParseRepositoryTag(config.Image) + v.Set("fromImage", repos) + v.Set("tag", tag) err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err) if err != nil { return err From e8db0311120d5e9a545fde5862f8a16b07760697 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 16:46:25 -0700 Subject: [PATCH 09/10] Fixed tag parsing when the repos name contains both a port and a tag --- tags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tags.go b/tags.go index f7307ed42..9ad9d10d0 100644 --- a/tags.go +++ b/tags.go @@ -70,11 +70,11 @@ func (store *TagStore) LookupImage(name string) (*Image, error) { if err != nil { // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else // (so we can pass all errors here) - repoAndTag := strings.SplitN(name, ":", 2) - if len(repoAndTag) == 1 { - repoAndTag = append(repoAndTag, DEFAULTTAG) + repos, tag := utils.ParseRepositoryTag(name) + if tag == "" { + tag = DEFAULTTAG } - if i, err := store.GetImage(repoAndTag[0], repoAndTag[1]); err != nil { + if i, err := store.GetImage(repos, tag); err != nil { return nil, err } else if i == nil { return nil, fmt.Errorf("Image does not exist: %s", name) From 316c8328aab8f9d518574b4bef9e537fed4185f6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Tue, 9 Jul 2013 16:46:55 -0700 Subject: [PATCH 10/10] Hardened repos name validation --- registry/registry.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/registry/registry.go b/registry/registry.go index 2f225aed9..fc84f19ec 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -67,7 +67,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") { + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return auth.IndexServerAddress(), reposName, err @@ -79,6 +79,12 @@ func ResolveRepositoryName(reposName string) (string, string, error) { } hostname := nameParts[0] reposName = nameParts[1] + if strings.Contains(hostname, "index.docker.io") { + return "", "", fmt.Errorf("Invalid repository name, try \"%s\" instead", reposName) + } + if err := validateRepositoryName(reposName); err != nil { + return "", "", err + } endpoint := fmt.Sprintf("https://%s/v1/", hostname) if err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err)