diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 51f2088..b6f2106 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -10,6 +10,14 @@ "Comment": "null-12", "Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9" }, + { + "ImportPath": "github.com/appc/docker2aci/lib", + "Rev": "073a801e328be8d6a6d4c9152cdd13ed88b615b4" + }, + { + "ImportPath": "github.com/appc/docker2aci/tarball", + "Rev": "073a801e328be8d6a6d4c9152cdd13ed88b615b4" + }, { "ImportPath": "github.com/appc/spec/aci", "Comment": "v0.3.0-2-g4c3cbeae4798", diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go new file mode 100644 index 0000000..53f1fa9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go @@ -0,0 +1,755 @@ +// Package docker2aci implements a simple library for converting docker images to +// App Container Images (ACIs). +package docker2aci + +import ( + "archive/tar" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/docker2aci/tarball" + "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/aci" + "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema" + "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema/types" +) + +const ( + defaultTag = "latest" + schemaVersion = "0.1.1" +) + +// Convert generates ACI images from docker registry URLs. +// It takes as input a dockerURL of the form: +// +// {docker registry URL}/{image name}:{tag} +// +// It then gets all the layers of the requested image and converts each of +// them to ACI. +// If the squash flag is true, it squashes all the layers in one file and +// places this file in outputDir; if it is false, it places every layer in its +// own ACI in outputDir. +// It returns the list of generated ACI paths. +func Convert(dockerURL string, squash bool, outputDir string) ([]string, error) { + parsedURL, err := parseDockerURL(dockerURL) + if err != nil { + return nil, fmt.Errorf("error parsing docker url: %v\n", err) + } + + repoData, err := getRepoData(parsedURL.IndexURL, parsedURL.ImageName) + if err != nil { + return nil, fmt.Errorf("error getting repository data: %v\n", err) + } + + // TODO(iaguis) check more endpoints + appImageID, err := getImageIDFromTag(repoData.Endpoints[0], parsedURL.ImageName, parsedURL.Tag, repoData) + if err != nil { + return nil, fmt.Errorf("error getting ImageID from tag %s: %v\n", parsedURL.Tag, err) + } + + ancestry, err := getAncestry(appImageID, repoData.Endpoints[0], repoData) + if err != nil { + return nil, fmt.Errorf("error getting ancestry: %v\n", err) + } + + layersOutputDir := outputDir + if squash { + layersOutputDir, err = ioutil.TempDir("", "docker2aci-") + if err != nil { + return nil, fmt.Errorf("error creating dir: %v", err) + } + defer os.RemoveAll(layersOutputDir) + } + + var aciLayerPaths []string + for i := len(ancestry) - 1; i >= 0; i-- { + layerID := ancestry[i] + aciPath, err := buildACI(layerID, repoData, parsedURL, layersOutputDir) + if err != nil { + return nil, fmt.Errorf("error building layer: %v\n", err) + } + + aciLayerPaths = append(aciLayerPaths, aciPath) + } + + if squash { + squashedFilename := strings.Replace(parsedURL.ImageName, "/", "-", -1) + if parsedURL.Tag != "" { + squashedFilename += "-" + parsedURL.Tag + } + squashedFilename += ".aci" + squashedImagePath := path.Join(outputDir, squashedFilename) + + if err := SquashLayers(aciLayerPaths, squashedImagePath); err != nil { + return nil, fmt.Errorf("error squashing image: %v\n", err) + } + aciLayerPaths = []string{squashedImagePath} + } + + return aciLayerPaths, nil +} + +func parseDockerURL(arg string) (*ParsedDockerURL, error) { + taglessRemote, tag := parseRepositoryTag(arg) + if tag == "" { + tag = defaultTag + } + indexURL, imageName := splitReposName(taglessRemote) + + return &ParsedDockerURL{ + IndexURL: indexURL, + ImageName: imageName, + Tag: tag, + }, nil +} + +func getRepoData(indexURL string, remote string) (*RepoData, error) { + client := &http.Client{} + repositoryURL := "https://" + path.Join(indexURL, "v1", "repositories", remote, "images") + + req, err := http.NewRequest("GET", repositoryURL, nil) + if err != nil { + return nil, err + } + + // TODO(iaguis) add auth? + req.Header.Set("X-Docker-Token", "true") + + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code: %d, URL: %s", res.StatusCode, req.URL) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + } + + var cookies []string + if res.Header.Get("Set-Cookie") != "" { + cookies = res.Header["Set-Cookie"] + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = makeEndpointsList(res.Header["X-Docker-Endpoints"]) + } else { + // Assume same endpoint + endpoints = append(endpoints, indexURL) + } + + return &RepoData{ + Endpoints: endpoints, + Tokens: tokens, + Cookie: cookies, + }, nil +} + +func getImageIDFromTag(registry string, appName string, tag string, repoData *RepoData) (string, error) { + client := &http.Client{} + req, err := http.NewRequest("GET", "https://"+path.Join(registry, "repositories", appName, "tags", tag), nil) + if err != nil { + return "", fmt.Errorf("failed to get Image ID: %s, URL: %s", err, req.URL) + } + + setAuthToken(req, repoData.Tokens) + setCookie(req, repoData.Cookie) + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to get Image ID: %s, URL: %s", err, req.URL) + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return "", fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + j, err := ioutil.ReadAll(res.Body) + + if err != nil { + return "", err + } + + var imageID string + + if err := json.Unmarshal(j, &imageID); err != nil { + return "", fmt.Errorf("error unmarshaling: %v", err) + } + + return imageID, nil +} + +func getAncestry(imgID, registry string, repoData *RepoData) ([]string, error) { + client := &http.Client{} + req, err := http.NewRequest("GET", "https://"+path.Join(registry, "images", imgID, "ancestry"), nil) + if err != nil { + return nil, err + } + + setAuthToken(req, repoData.Tokens) + setCookie(req, repoData.Cookie) + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + var ancestry []string + + j, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Failed to read downloaded json: %s (%s)", err, j) + } + + if err := json.Unmarshal(j, &ancestry); err != nil { + return nil, fmt.Errorf("error unmarshaling: %v", err) + } + + return ancestry, nil +} + +func buildACI(layerID string, repoData *RepoData, dockerURL *ParsedDockerURL, outputDir string) (string, error) { + tmpDir, err := ioutil.TempDir("", "docker2aci-") + if err != nil { + return "", fmt.Errorf("error creating dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + layerDest := filepath.Join(tmpDir, "layer") + layerRootfs := filepath.Join(layerDest, "rootfs") + err = os.MkdirAll(layerRootfs, 0700) + if err != nil { + return "", fmt.Errorf("error creating dir: %s", layerRootfs) + } + + j, size, err := getRemoteImageJSON(layerID, repoData.Endpoints[0], repoData) + if err != nil { + return "", fmt.Errorf("error getting image json: %v", err) + } + + layerData := DockerImageData{} + if err := json.Unmarshal(j, &layerData); err != nil { + return "", fmt.Errorf("error unmarshaling layer data: %v", err) + } + + layer, err := getRemoteLayer(layerID, repoData.Endpoints[0], repoData, int64(size)) + if err != nil { + return "", fmt.Errorf("error getting the remote layer: %v", err) + } + defer layer.Close() + + layerFile, err := ioutil.TempFile(tmpDir, "dockerlayer-") + if err != nil { + return "", fmt.Errorf("error creating layer: %v", err) + } + + _, err = io.Copy(layerFile, layer) + if err != nil { + return "", fmt.Errorf("error getting layer: %v", err) + } + + layerFile.Sync() + + manifest, err := generateManifest(layerData, dockerURL) + if err != nil { + return "", fmt.Errorf("error generating the manifest: %v", err) + } + + imageName := strings.Replace(dockerURL.ImageName, "/", "-", -1) + aciPath := imageName + "-" + layerID + if dockerURL.Tag != "" { + aciPath += "-" + dockerURL.Tag + } + if layerData.OS != "" { + aciPath += "-" + layerData.OS + if layerData.Architecture != "" { + aciPath += "-" + layerData.Architecture + } + } + aciPath += ".aci" + + aciPath = path.Join(outputDir, aciPath) + + if err := writeACI(layerFile, *manifest, aciPath); err != nil { + return "", fmt.Errorf("error writing ACI: %v", err) + } + + if err := validateACI(aciPath); err != nil { + return "", fmt.Errorf("invalid aci generated: %v", err) + } + + return aciPath, nil +} + +func validateACI(aciPath string) error { + aciFile, err := os.Open(aciPath) + if err != nil { + return err + } + defer aciFile.Close() + + reader, err := aci.NewCompressedTarReader(aciFile) + if err != nil { + return err + } + + if err := aci.ValidateArchive(reader); err != nil { + return err + } + + return nil +} + +func getRemoteImageJSON(imgID, registry string, repoData *RepoData) ([]byte, int, error) { + client := &http.Client{} + req, err := http.NewRequest("GET", "https://"+path.Join(registry, "images", imgID, "json"), nil) + if err != nil { + return nil, -1, err + } + setAuthToken(req, repoData.Tokens) + setCookie(req, repoData.Cookie) + res, err := client.Do(req) + if err != nil { + return nil, -1, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, -1, fmt.Errorf("HTTP code: %d, URL: %s", res.StatusCode, req.URL) + } + + imageSize := -1 + + if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { + imageSize, err = strconv.Atoi(hdr) + if err != nil { + return nil, -1, err + } + } + + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, -1, fmt.Errorf("failed to read downloaded json: %v (%s)", err, b) + } + + return b, imageSize, nil +} + +func getRemoteLayer(imgID, registry string, repoData *RepoData, imgSize int64) (io.ReadCloser, error) { + client := &http.Client{} + req, err := http.NewRequest("GET", "https://"+path.Join(registry, "images", imgID, "layer"), nil) + if err != nil { + return nil, err + } + + setAuthToken(req, repoData.Tokens) + setCookie(req, repoData.Cookie) + + fmt.Printf("Downloading layer: %s\n", imgID) + + res, err := client.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode != 200 { + res.Body.Close() + return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + return res.Body, nil +} + +func generateManifest(layerData DockerImageData, dockerURL *ParsedDockerURL) (*schema.ImageManifest, error) { + dockerConfig := layerData.Config + genManifest := &schema.ImageManifest{} + + appURL := dockerURL.IndexURL + "/" + dockerURL.ImageName + "-" + layerData.ID + name, err := types.NewACName(appURL) + if err != nil { + return nil, err + } + genManifest.Name = *name + + acVersion, _ := types.NewSemVer(schemaVersion) + genManifest.ACVersion = *acVersion + + genManifest.ACKind = types.ACKind("ImageManifest") + + var labels types.Labels + var parentLabels types.Labels + + layer, _ := types.NewACName("layer") + labels = append(labels, types.Label{Name: *layer, Value: layerData.ID}) + + tag := dockerURL.Tag + version, _ := types.NewACName("version") + labels = append(labels, types.Label{Name: *version, Value: tag}) + + if layerData.OS != "" { + os, _ := types.NewACName("os") + labels = append(labels, types.Label{Name: *os, Value: layerData.OS}) + parentLabels = append(parentLabels, types.Label{Name: *os, Value: layerData.OS}) + + if layerData.Architecture != "" { + arch, _ := types.NewACName("arch") + parentLabels = append(parentLabels, types.Label{Name: *arch, Value: layerData.Architecture}) + } + } + + genManifest.Labels = labels + + if dockerConfig != nil { + var exec types.Exec + if len(dockerConfig.Cmd) > 0 { + exec = types.Exec(dockerConfig.Cmd) + } else if len(dockerConfig.Entrypoint) > 0 { + exec = types.Exec(dockerConfig.Entrypoint) + } + if exec != nil { + user, group := parseDockerUser(dockerConfig.User) + app := &types.App{Exec: exec, User: user, Group: group} + genManifest.App = app + } + } + + if layerData.Parent != "" { + var dependencies types.Dependencies + parentAppNameString := dockerURL.IndexURL + "/" + dockerURL.ImageName + "-" + layerData.Parent + + parentAppName, err := types.NewACName(parentAppNameString) + if err != nil { + return nil, err + } + + dependencies = append(dependencies, types.Dependency{App: *parentAppName, Labels: parentLabels}) + + genManifest.Dependencies = dependencies + } + + return genManifest, nil +} + +func parseDockerUser(dockerUser string) (string, string) { + // if the docker user is empty assume root user and group + if dockerUser == "" { + return "0", "0" + } + + dockerUserParts := strings.Split(dockerUser, ":") + + // when only the user is given, the docker spec says that the default and + // supplementary groups of the user in /etc/passwd should be applied. + // Assume root group for now in this case. + if len(dockerUserParts) < 2 { + return dockerUserParts[0], "0" + } + + return dockerUserParts[0], dockerUserParts[1] +} + +func writeACI(layer io.ReadSeeker, manifest schema.ImageManifest, output string) error { + reader, err := aci.NewCompressedTarReader(layer) + if err != nil { + return err + } + + aciFile, err := os.Create(output) + if err != nil { + return fmt.Errorf("error creating ACI file: %v", err) + } + defer aciFile.Close() + + trw := tar.NewWriter(aciFile) + defer trw.Close() + + if err := addMinimalACIStructure(trw, manifest); err != nil { + return fmt.Errorf("error writing rootfs entry: %v", err) + } + + // Write files in rootfs/ + if err = tarball.Walk(*reader, func(t *tarball.TarFile) error { + name := t.Name() + if name == "./" { + return nil + } + t.Header.Name = path.Join("rootfs", name) + if strings.Contains(t.Header.Name, "/.wh.") { + return nil + } + if t.Header.Typeflag == tar.TypeLink { + t.Header.Linkname = path.Join("rootfs" + t.Linkname()) + } + + if err := trw.WriteHeader(t.Header); err != nil { + return err + } + if _, err := io.Copy(trw, t.TarStream); err != nil { + return err + } + + return nil + }); err != nil { + return err + } + + return nil +} + +func addMinimalACIStructure(tarWriter *tar.Writer, manifest schema.ImageManifest) error { + hdr := getGenericTarHeader() + hdr.Name = "rootfs" + hdr.Mode = 0755 + hdr.Size = int64(0) + hdr.Typeflag = tar.TypeDir + + if err := tarWriter.WriteHeader(hdr); err != nil { + return err + } + + writeManifest(tarWriter, manifest) + + return nil +} + +func getGenericTarHeader() *tar.Header { + // FIXME(iaguis) Use docker image time instead of the Unix Epoch? + hdr := &tar.Header{ + Uid: 0, + Gid: 0, + ModTime: time.Unix(0, 0), + Uname: "0", + Gname: "0", + ChangeTime: time.Unix(0, 0), + } + + return hdr +} + +func writeManifest(outputWriter *tar.Writer, manifest schema.ImageManifest) error { + b, err := json.Marshal(manifest) + if err != nil { + return err + } + + hdr := getGenericTarHeader() + hdr.Name = "manifest" + hdr.Mode = 0644 + hdr.Size = int64(len(b)) + hdr.Typeflag = tar.TypeReg + + if err := outputWriter.WriteHeader(hdr); err != nil { + return err + } + if _, err := outputWriter.Write(b); err != nil { + return err + } + + return nil +} + +// SquashLayers receives a list of ACI layer file names ordered from base image +// to application image and squashes them into one ACI +func SquashLayers(layers []string, squashedImagePath string) error { + manifests, err := getManifests(layers) + if err != nil { + return err + } + + fileMap, err := getFilesToLayersMap(layers) + if err != nil { + return err + } + + squashedImageFile, err := os.Create(squashedImagePath) + if err != nil { + return err + } + defer squashedImageFile.Close() + + if err := writeSquashedImage(squashedImageFile, layers, fileMap, manifests); err != nil { + return err + } + + if err := validateACI(squashedImagePath); err != nil { + return err + } + + return nil +} + +func getManifests(layers []string) ([]schema.ImageManifest, error) { + var manifests []schema.ImageManifest + + for _, aciPath := range layers { + currentFile, err := os.Open(aciPath) + if err != nil { + return nil, err + } + defer currentFile.Close() + + manifestCur, err := aci.ManifestFromImage(currentFile) + if err != nil { + return nil, err + } + if _, err := currentFile.Seek(0, os.SEEK_SET); err != nil { + return nil, err + } + + manifests = append(manifests, *manifestCur) + } + + return manifests, nil +} + +func getFilesToLayersMap(layers []string) (map[string]string, error) { + var err error + fileMap := make(map[string]string) + for _, aciPath := range layers { + fileMap, err = gatherFilesToLayersMap(fileMap, aciPath) + if err != nil { + return nil, err + } + } + + return fileMap, nil +} + +// gatherFilesToLayersMap accumulates a map associationg each file of the final +// image with the layer it comes from. It should be called starting from the +// base layer so that the order of files in the squashed layer is preserved +// and, if a file is present several times, the last layer is taken into +// account. +func gatherFilesToLayersMap(fileMap map[string]string, currentPath string) (map[string]string, error) { + currentFile, err := os.Open(currentPath) + if err != nil { + return nil, err + } + defer currentFile.Close() + + reader, err := aci.NewCompressedTarReader(currentFile) + if err != nil { + return nil, err + } + + if err = tarball.Walk(*reader, func(t *tarball.TarFile) error { + if t.Name() == "manifest" { + return nil + } + + fileMap[t.Name()] = currentPath + return nil + }); err != nil { + return nil, err + } + + return fileMap, nil +} + +func writeSquashedImage(outputFile *os.File, layers []string, fileMap map[string]string, manifests []schema.ImageManifest) error { + outputWriter := tar.NewWriter(outputFile) + defer outputWriter.Close() + + var err error + for _, aciPath := range layers { + outputWriter, err = reduceACIs(outputWriter, fileMap, aciPath) + if err != nil { + return err + } + } + + finalManifest := mergeManifests(manifests) + + if err := writeManifest(outputWriter, finalManifest); err != nil { + return err + } + + return nil +} + +func reduceACIs(outputWriter *tar.Writer, fileMap map[string]string, currentPath string) (*tar.Writer, error) { + currentFile, err := os.Open(currentPath) + if err != nil { + return nil, err + } + defer currentFile.Close() + + reader, err := aci.NewCompressedTarReader(currentFile) + if err != nil { + return nil, err + } + if err = tarball.Walk(*reader, func(t *tarball.TarFile) error { + if t.Name() == "manifest" { + return nil + } + + if fileMap[t.Name()] == currentPath { + if err := outputWriter.WriteHeader(t.Header); err != nil { + return fmt.Errorf("Error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("Error copying file into the tar out: %v", err) + } + } + + return nil + }); err != nil { + return nil, err + } + + return outputWriter, nil +} + +func mergeManifests(manifests []schema.ImageManifest) schema.ImageManifest { + // FIXME(iaguis) we take last layer's manifest as the final manifest for now + manifest := manifests[len(manifests)-1] + + manifest.Dependencies = nil + + layerIndex := -1 + for i, l := range manifest.Labels { + if l.Name.String() == "layer" { + layerIndex = i + } + } + + if layerIndex != -1 { + manifest.Labels = append(manifest.Labels[:layerIndex], manifest.Labels[layerIndex+1:]...) + } + + // this can't fail because the old name is legal + nameWithoutLayerID, _ := types.NewACName(strings.Split(manifest.Name.String(), "-")[0]) + + manifest.Name = *nameWithoutLayerID + + return manifest +} + +func setAuthToken(req *http.Request, token []string) { + if req.Header.Get("Authorization") == "" { + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } +} + +func setCookie(req *http.Request, cookie []string) { + if req.Header.Get("Cookie") == "" { + req.Header.Set("Cookie", strings.Join(cookie, "")) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker_functions.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker_functions.go new file mode 100644 index 0000000..9a316e3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker_functions.go @@ -0,0 +1,40 @@ +package docker2aci + +import ( + "strings" +) + +const ( + defaultIndex = "index.docker.io" +) + +// splitReposName breaks a reposName into an index name and remote name +func splitReposName(reposName string) (string, string) { + nameParts := strings.SplitN(reposName, "/", 2) + var indexName, remoteName string + if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && + !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + // 'docker.io' + indexName = defaultIndex + remoteName = reposName + } else { + indexName = nameParts[0] + remoteName = nameParts[1] + } + return indexName, remoteName +} + +// Get a repos name and returns the right reposName + tag +// The tag can be confusing because of a port in a repository name. +// Ex: localhost.localdomain:5000/samalba/hipache:latest +func parseRepositoryTag(repos string) (string, string) { + n := strings.LastIndex(repos, ":") + if n < 0 { + return repos, "" + } + if tag := repos[n+1:]; !strings.Contains(tag, "/") { + return repos[:n], tag + } + return repos, "" +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker_types.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker_types.go new file mode 100644 index 0000000..3b45652 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker_types.go @@ -0,0 +1,51 @@ +package docker2aci + +import "time" + +// DockerImageData stores the JSON structure of a Docker image. +// Taken and adapted from upstream Docker. +type DockerImageData struct { + ID string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + Container string `json:"container,omitempty"` + ContainerConfig DockerImageConfig `json:"container_config,omitempty"` + DockerVersion string `json:"docker_version,omitempty"` + Author string `json:"author,omitempty"` + Config *DockerImageConfig `json:"config,omitempty"` + Architecture string `json:"architecture,omitempty"` + OS string `json:"os,omitempty"` + Checksum string `json:"checksum"` +} + +// Note: the Config structure should hold only portable information about the container. +// Here, "portable" means "independent from the host we are running on". +// Non-portable information *should* appear in HostConfig. +// Taken and adapted from upstream Docker. +type DockerImageConfig struct { + Hostname string + Domainname string + User string + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + CpuShares int64 // CPU shares (relative weight vs. other containers) + Cpuset string // Cpuset 0-2, 0,1 + AttachStdin bool + AttachStdout bool + AttachStderr bool + PortSpecs []string // Deprecated - Can be in the format of 8080/tcp + ExposedPorts map[string]struct{} + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin + StdinOnce bool // If true, close stdin after the 1 attached client disconnects. + Env []string + Cmd []string + Image string // Name of the image as it was passed by the operator (eg. could be symbolic) + Volumes map[string]struct{} + WorkingDir string + Entrypoint []string + NetworkDisabled bool + MacAddress string + OnBuild []string +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/types.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/types.go new file mode 100644 index 0000000..f591adb --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/types.go @@ -0,0 +1,13 @@ +package docker2aci + +type RepoData struct { + Tokens []string + Endpoints []string + Cookie []string +} + +type ParsedDockerURL struct { + IndexURL string + ImageName string + Tag string +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/util.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/util.go new file mode 100644 index 0000000..bd4bacc --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/util.go @@ -0,0 +1,22 @@ +package docker2aci + +import ( + "path" + "strings" +) + +func makeEndpointsList(headers []string) []string { + var endpoints []string + + for _, ep := range headers { + endpointsList := strings.Split(ep, ",") + for _, endpointEl := range endpointsList { + endpoints = append( + endpoints, + // TODO(iaguis) discover if httpsOrHTTP + path.Join(strings.TrimSpace(endpointEl), "v1")) + } + } + + return endpoints +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/tarball/tarfile.go b/Godeps/_workspace/src/github.com/appc/docker2aci/tarball/tarfile.go new file mode 100644 index 0000000..db5ce66 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/tarball/tarfile.go @@ -0,0 +1,24 @@ +package tarball + +import ( + "archive/tar" + "io" +) + +// TarFile is a representation of a file in a tarball. It consists of two parts, +// the Header and the Stream. The Header is a regular tar header, the Stream +// is a byte stream that can be used to read the file's contents +type TarFile struct { + Header *tar.Header + TarStream io.Reader +} + +// Name returns the name of the file as reported by the header +func (t *TarFile) Name() string { + return t.Header.Name +} + +// Linkname returns the Linkname of the file as reported by the header +func (t *TarFile) Linkname() string { + return t.Header.Linkname +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/tarball/walk.go b/Godeps/_workspace/src/github.com/appc/docker2aci/tarball/walk.go new file mode 100644 index 0000000..1d834b5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/tarball/walk.go @@ -0,0 +1,29 @@ +package tarball + +import ( + "archive/tar" + "fmt" + "io" +) + +// WalkFunc is a func for handling each file (header and byte stream) in a tarball +type WalkFunc func(t *TarFile) error + +// Walk walks through the files in the tarball represented by tarstream and +// passes each of them to the WalkFunc provided as an argument +func Walk(tarReader tar.Reader, walkFunc func(t *TarFile) error) error { + for { + hdr, err := tarReader.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return fmt.Errorf("Error reading tar entry: %v", err) + } + if err := walkFunc(&TarFile{Header: hdr, TarStream: &tarReader}); err != nil { + return err + } + } + return nil +} diff --git a/cas/cas.go b/cas/cas.go index b00fb52..aee0b28 100644 --- a/cas/cas.go +++ b/cas/cas.go @@ -74,13 +74,21 @@ func NewStore(base string) *Store { } func (ds Store) tmpFile() (*os.File, error) { - dir := filepath.Join(ds.base, "tmp") - if err := os.MkdirAll(dir, defaultPathPerm); err != nil { + dir, err := ds.tmpDir() + if err != nil { return nil, err } return ioutil.TempFile(dir, "") } +func (ds Store) tmpDir() (string, error) { + dir := filepath.Join(ds.base, "tmp") + if err := os.MkdirAll(dir, defaultPathPerm); err != nil { + return "", err + } + return dir, nil +} + // ResolveKey resolves a partial key (of format `sha512-0c45e8c0ab2`) to a full // key by considering the key a prefix and using the store for resolution. // If the key is longer than the full key length, it is first truncated. diff --git a/cas/cas_test.go b/cas/cas_test.go index 8c41580..e9a612b 100644 --- a/cas/cas_test.go +++ b/cas/cas_test.go @@ -94,8 +94,8 @@ func TestDownloading(t *testing.T) { hit bool }{ // The Blob entry isn't used - {Remote{ts.URL, "", "12", ""}, body, false}, - {Remote{ts.URL, "", "12", ""}, body, true}, + {Remote{ts.URL, "", "", "12", ""}, body, false}, + {Remote{ts.URL, "", "", "12", ""}, body, true}, } ds := NewStore(dir) diff --git a/cas/remote.go b/cas/remote.go index 3d5991d..6188266 100644 --- a/cas/remote.go +++ b/cas/remote.go @@ -24,10 +24,12 @@ import ( "io/ioutil" "net/http" "os" + "strings" "time" "github.com/coreos/rocket/pkg/keystore" + "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/docker2aci/lib" "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/aci" "github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema/types" "github.com/coreos/rocket/Godeps/_workspace/src/github.com/mitchellh/ioprogress" @@ -44,6 +46,8 @@ func NewRemote(aciurl, sigurl string) *Remote { type Remote struct { ACIURL string + // Currently must be either empty or "docker" + Scheme string SigURL string ETag string // The key in the blob store under which the ACI has been saved. @@ -77,6 +81,27 @@ func (r Remote) Type() int64 { func (r Remote) Download(ds Store, ks *keystore.Keystore) (*openpgp.Entity, *os.File, error) { var entity *openpgp.Entity var err error + if r.Scheme == "docker" { + registryURL := strings.TrimPrefix(r.ACIURL, "docker://") + + tmpDir, err := ds.tmpDir() + if err != nil { + return nil, nil, fmt.Errorf("error creating temporary dir for docker to ACI conversion: %v", err) + } + + acis, err := docker2aci.Convert(registryURL, true, tmpDir) + if err != nil { + return nil, nil, fmt.Errorf("error converting docker image to ACI: %v", err) + } + + aciFile, err := os.Open(acis[0]) + if err != nil { + return nil, nil, fmt.Errorf("error opening squashed ACI file: %v", err) + } + + return nil, aciFile, nil + } + acif, err := downloadACI(ds, r.ACIURL) if err != nil { return nil, acif, fmt.Errorf("error downloading the aci image: %v", err) diff --git a/rkt/fetch.go b/rkt/fetch.go index 9031f8f..f19daad 100644 --- a/rkt/fetch.go +++ b/rkt/fetch.go @@ -91,10 +91,12 @@ func fetchImage(img string, ds *cas.Store, ks *keystore.Keystore, discover bool) if err != nil { return "", fmt.Errorf("not a valid URL (%s)", img) } - if u.Scheme != "http" && u.Scheme != "https" { - return "", fmt.Errorf("rkt only supports http or https URLs (%s)", img) + switch u.Scheme { + case "http", "https", "docker": + default: + return "", fmt.Errorf("rkt only supports http, https or docker URLs (%s)", img) } - return fetchImageFromURL(u.String(), ds, ks) + return fetchImageFromURL(u.String(), u.Scheme, ds, ks) } func fetchImageFromEndpoints(ep *discovery.Endpoints, ds *cas.Store, ks *keystore.Keystore) (string, error) { @@ -102,8 +104,9 @@ func fetchImageFromEndpoints(ep *discovery.Endpoints, ds *cas.Store, ks *keystor return downloadImage(rem, ds, ks) } -func fetchImageFromURL(imgurl string, ds *cas.Store, ks *keystore.Keystore) (string, error) { +func fetchImageFromURL(imgurl string, scheme string, ds *cas.Store, ks *keystore.Keystore) (string, error) { rem := cas.NewRemote(imgurl, sigURLFromImgURL(imgurl)) + rem.Scheme = scheme return downloadImage(rem, ds, ks) } @@ -112,6 +115,9 @@ func downloadImage(rem *cas.Remote, ds *cas.Store, ks *keystore.Keystore) (strin if globalFlags.InsecureSkipVerify { stdout("rkt: warning: signature verification has been disabled") } + if rem.Scheme == "docker" { + fmt.Printf("rkt: warning: signature verification for docker images is not supported\n") + } err := ds.ReadIndex(rem) if err != nil && rem.BlobKey == "" { entity, aciFile, err := rem.Download(*ds, ks) @@ -120,7 +126,7 @@ func downloadImage(rem *cas.Remote, ds *cas.Store, ks *keystore.Keystore) (strin } defer os.Remove(aciFile.Name()) - if !globalFlags.InsecureSkipVerify { + if entity != nil && !globalFlags.InsecureSkipVerify { fmt.Println("rkt: signature verified: ") for _, v := range entity.Identities { stdout(" %s", v.Name)