diff --git a/README.md b/README.md index 18e9b85..34d9c7b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Keep in mind while running through the examples that rkt needs to be ran as root Rocket uses content addressable storage (CAS) for storing an ACI on disk. In this example, the image is downloaded and added to the CAS. ``` -$ rkt fetch https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace_validator_main.tar.gz +$ rkt fetch https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace-validator-main.tar.gz sha256-f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ``` @@ -47,11 +47,11 @@ $ find /var/lib/rkt/cas/object/ Per the App Container [spec][spec] the sha256 is of the tarball, which is reproducable with other tools: ``` -$ wget https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace_validator_main.tar.gz +$ wget https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace-validator-main.tar.gz ... -$ gunzip ace_validator_main.tar.gz -$ sha256sum ace_validator_main.tar -f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ace_validator_main.tar +$ gunzip ace-validator-main.tar.gz +$ sha256sum ace-validator-main.tar +f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ace-validator-main.tar ``` ### Launching an ACI @@ -59,7 +59,7 @@ f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ace_validator_ To run an ACI, you can either use the sha256 hash, or the URL which you downloaded it from: ``` -$ rkt run https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace_validator_main.tar.gz +$ rkt run https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace-validator-main.tar.gz ``` rkt will do the appropriate etag checking on the URL to make sure it has the most up to date version of the image. diff --git a/app-container/SPEC.md b/app-container/SPEC.md index 9ba5fd0..7893960 100644 --- a/app-container/SPEC.md +++ b/app-container/SPEC.md @@ -259,8 +259,8 @@ Then inspect the HTML returned for meta tags that have the following format: Some examples for different schemes and URLs: ``` - - + + ``` diff --git a/app-container/ace/app_manifest_main.json b/app-container/ace/app_manifest_main.json index 23107f0..1e80943 100644 --- a/app-container/ace/app_manifest_main.json +++ b/app-container/ace/app_manifest_main.json @@ -5,19 +5,19 @@ "os": "linux", "arch": "amd64", "exec": [ - "/ace_validator", "main" + "/ace-validator", "main" ], "eventHandlers": [ { "name": "pre-start", "exec": [ - "/ace_validator", "prestart" + "/ace-validator", "prestart" ] }, { "name": "post-stop", "exec": [ - "/ace_validator", "poststop" + "/ace-validator", "poststop" ] } ], diff --git a/app-container/ace/app_manifest_sidekick.json b/app-container/ace/app_manifest_sidekick.json index 6381f81..1941731 100644 --- a/app-container/ace/app_manifest_sidekick.json +++ b/app-container/ace/app_manifest_sidekick.json @@ -5,7 +5,7 @@ "os": "linux", "arch": "amd64", "exec": [ - "/ace_validator", "sidekick" + "/ace-validator", "sidekick" ], "user": "0", "group": "0", diff --git a/app-container/ace/build_aci b/app-container/ace/build_aci index 5ad9ee2..f351143 100755 --- a/app-container/ace/build_aci +++ b/app-container/ace/build_aci @@ -13,25 +13,25 @@ fi for typ in main sidekick; do layoutdir="bin/ace_${typ}_layout" mkdir -p ${layoutdir}/rootfs - cp bin/ace_validator ${layoutdir}/rootfs/ + cp bin/ace-validator ${layoutdir}/rootfs/ cp ${PREFIX}/app_manifest_${typ}.json ${layoutdir}/app # now build the tarball, and sign it pushd ${layoutdir} >/dev/null # Set a consistent timestamp so we get a consistent hash # TODO(jonboulle): make this cleaner.. - for path in rootfs rootfs/ace_validator; do + for path in rootfs rootfs/ace-validator; do touch -a -m -d 1415660606 ${path} done - ../actool build --overwrite --app-manifest app rootfs/ ../ace_validator_${typ}.aci - HASH=sha256-$(sha256sum ../ace_validator_${typ}.aci|awk '{print $1}') - gzip -f ../ace_validator_${typ}.aci - mv ../ace_validator_${typ}.aci.gz ../ace_validator_${typ}.aci - gpg --cipher-algo AES256 --output ace_validator_${typ}.sig --detach-sig ../ace_validator_${typ}.aci - mv ace_validator_${typ}.sig ../ + ../actool build --overwrite --app-manifest app rootfs/ ../ace-validator-${typ}.aci + HASH=sha256-$(sha256sum ../ace-validator-${typ}.aci|awk '{print $1}') + gzip -f ../ace-validator-${typ}.aci + mv ../ace-validator-${typ}.aci.gz ../ace-validator-${typ}.aci + gpg --cipher-algo AES256 --output ace-validator-${typ}.sig --detach-sig ../ace-validator-${typ}.aci + mv ace-validator-${typ}.sig ../ popd >/dev/null echo "Wrote ${typ} layout to ${layoutdir}" - echo "Wrote unsigned ${typ} ACI bin/ace_validator_${typ}.aci" - ln -s ${PWD}/bin/ace_validator_${typ}.aci bin/${HASH} + echo "Wrote unsigned ${typ} ACI bin/ace-validator-${typ}.aci" + ln -s ${PWD}/bin/ace-validator-${typ}.aci bin/${HASH} echo "Wrote ${typ} layout hash bin/${HASH}" - echo "Wrote ${typ} ACI signature bin/ace_validator_${typ}.sig" + echo "Wrote ${typ} ACI signature bin/ace-validator-${typ}.sig" done diff --git a/app-container/actool/discover.go b/app-container/actool/discover.go index 4e44844..e5cd0b3 100644 --- a/app-container/actool/discover.go +++ b/app-container/actool/discover.go @@ -30,12 +30,12 @@ func runDiscover(args []string) (exit int) { q := globalFlags.Quiet for _, name := range args { - labels, err := appFromString(name) + app, err := discovery.NewAppFromString(name) if err != nil { stderr(q, "%s: %s", name, err) return 1 } - eps, err := discovery.DiscoverEndpoints(labels["name"], labels["ver"], labels["os"], labels["amd64"], transportFlags.Insecure) + eps, err := discovery.DiscoverEndpoints(*app, transportFlags.Insecure) if err != nil { stderr(q, "error fetching %s: %s", name, err) diff --git a/app-container/actool/fetch.go b/app-container/actool/fetch.go deleted file mode 100644 index 6144617..0000000 --- a/app-container/actool/fetch.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/coreos/rocket/app-container/discovery" -) - -var ( - cmdFetch = &Command{ - Name: "fetch", - Description: "Discover and download an app container image", - Summary: "Discover, download and store on disk the app container image for one or more apps", - Run: runFetch, - } -) - -func runFetch(args []string) (exit int) { - if len(args) < 1 { - fmt.Fprintf(os.Stderr, "discover: at least one name required") - } - q := globalFlags.Quiet - - for _, name := range args { - labels, err := appFromString(name) - if err != nil { - stderr(q, "%s: %s", name, err) - return 1 - } - eps, err := discovery.DiscoverEndpoints(labels["name"], labels["ver"], labels["os"], labels["arch"], transportFlags.Insecure) - if err != nil { - stderr(q, "error fetching %s: %s", name, err) - return 1 - } - // TODO(philips): store the images.. - fmt.Println(strings.Join(eps.Sig, ",")) - fmt.Println(strings.Join(eps.ACI, ",")) - fmt.Println(strings.Join(eps.Keys, ",")) - } - - return -} diff --git a/app-container/actool/util.go b/app-container/actool/util.go deleted file mode 100644 index e0ed5b1..0000000 --- a/app-container/actool/util.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "fmt" - "net/url" - "runtime" - "strings" -) - -const ( - defaultVersion = "latest" - defaultOS = runtime.GOOS - defaultArch = runtime.GOARCH -) - -// appFromString takes a command line app parameter and returns a map of labels. -// -// Example app parameters: -// example.com/reduce-worker:1.0.0 -// example.com/reduce-worker,channel=alpha,label=value -func appFromString(app string) (out map[string]string, err error) { - out = make(map[string]string, 0) - app = strings.Replace(app, ":", ",ver=", -1) - app = "name=" + app - v, err := url.ParseQuery(strings.Replace(app, ",", "&", -1)) - if err != nil { - return nil, err - } - for key, val := range v { - if len(val) > 1 { - return nil, fmt.Errorf("label %s with multiple values %q", key, val) - } - out[key] = val[0] - } - if out["ver"] == "" { - out["ver"] = defaultVersion - } - if out["os"] == "" { - out["os"] = defaultOS - } - if out["arch"] == "" { - out["arch"] = defaultArch - } - return -} diff --git a/app-container/discovery/discovery.go b/app-container/discovery/discovery.go index 1083fa1..f3c6f8f 100644 --- a/app-container/discovery/discovery.go +++ b/app-container/discovery/discovery.go @@ -78,8 +78,8 @@ func renderTemplate(tpl string, kvs ...string) string { return tpl } -func DiscoverEndpoints(app, ver, os, arch string, insecure bool) (*Endpoints, error) { - _, body, err := httpsOrHTTP(app, insecure) +func DiscoverEndpoints(app App, insecure bool) (*Endpoints, error) { + _, body, err := httpsOrHTTP(app.Name.String(), insecure) if err != nil { return nil, err } @@ -87,13 +87,13 @@ func DiscoverEndpoints(app, ver, os, arch string, insecure bool) (*Endpoints, er meta := extractACMeta(body) - appver := app + "-" + ver - tplVars := []string{"{os}", os, "{arch}", arch, "{app-ver}", appver} + tplVars := []string{"{os}", app.Labels["os"], "{arch}", app.Labels["arch"], + "{name}", app.Name.String(), "{version}", app.Labels["version"]} de := &Endpoints{} for _, m := range meta { - if !strings.HasPrefix(appver, m.prefix) { + if !strings.HasPrefix(app.Name.String(), m.prefix) { continue } diff --git a/app-container/discovery/parse.go b/app-container/discovery/parse.go new file mode 100644 index 0000000..aa8d41a --- /dev/null +++ b/app-container/discovery/parse.go @@ -0,0 +1,80 @@ +package discovery + +import ( + "fmt" + "net/url" + "runtime" + "strings" + + "github.com/coreos/rocket/app-container/schema/types" +) + +const ( + defaultVersion = "latest" + defaultOS = runtime.GOOS + defaultArch = runtime.GOARCH +) + +type App struct { + Name types.ACName + Labels map[string]string +} + +func NewApp(name string, labels map[string]string) (*App, error) { + if labels == nil { + labels = make(map[string]string, 0) + } + acn, err := types.NewACName(name) + if err != nil { + return nil, err + } + return &App{ + Name: *acn, + Labels: labels, + }, nil +} + +// NewAppFromString takes a command line app parameter and returns a map of labels. +// +// Example app parameters: +// example.com/reduce-worker:1.0.0 +// example.com/reduce-worker,channel=alpha,label=value +func NewAppFromString(app string) (*App, error) { + var ( + name string + labels map[string]string + ) + + app = strings.Replace(app, ":", ",version=", -1) + app = "name=" + app + v, err := url.ParseQuery(strings.Replace(app, ",", "&", -1)) + if err != nil { + return nil, err + } + labels = make(map[string]string, 0) + for key, val := range v { + if len(val) > 1 { + return nil, fmt.Errorf("label %s with multiple values %q", key, val) + } + if key == "name" { + name = val[0] + continue + } + labels[key] = val[0] + } + if labels["version"] == "" { + labels["version"] = defaultVersion + } + if labels["os"] == "" { + labels["os"] = defaultOS + } + if labels["arch"] == "" { + labels["arch"] = defaultArch + } + + a, err := NewApp(name, labels) + if err != nil { + return nil, err + } + return a, nil +} diff --git a/build b/build index 2984d4a..e349ab6 100755 --- a/build +++ b/build @@ -17,7 +17,7 @@ echo "Building actool..." go build -o $GOBIN/actool ${REPO_PATH}/app-container/actool echo "Building ACE validator..." -GOOS=linux CGO_ENABLED=0 go build -a -ldflags '-extldflags "-static"' -o $GOBIN/ace_validator ${REPO_PATH}/app-container/ace +GOOS=linux CGO_ENABLED=0 go build -a -ldflags '-extldflags "-static"' -o $GOBIN/ace-validator ${REPO_PATH}/app-container/ace echo "Building init (stage1)..." go build -o $GOBIN/init ${REPO_PATH}/stage1 diff --git a/rkt/fetch.go b/rkt/fetch.go index bd55c9c..e72e89e 100644 --- a/rkt/fetch.go +++ b/rkt/fetch.go @@ -2,9 +2,11 @@ package main import ( "fmt" + "net/url" "os" "path/filepath" + "github.com/coreos/rocket/app-container/discovery" "github.com/coreos/rocket/cas" ) @@ -33,6 +35,39 @@ func fetchURL(img string, ds *cas.Store) (string, error) { return rem.Blob, nil } +// fetchImage will take an image as either a URL or a name string and import it +// into the store if found. +func fetchImage(img string, ds *cas.Store) (string, error) { + // discover if it isn't a URL + u, err := url.Parse(img) + if err == nil && u.Scheme == "" { + app, err := discovery.NewAppFromString(img) + if globalFlags.Debug && err != nil { + fmt.Printf("discovery: %s\n", err) + } + if err == nil { + ep, err := discovery.DiscoverEndpoints(*app, true) + if err != nil { + return "", err + } + // TODO(philips): use all available mirrors + if globalFlags.Debug { + fmt.Printf("fetch: trying %v\n", ep.ACI) + } + img = ep.ACI[0] + u, err = url.Parse(img) + } + } + + if err != nil { // download if it isn't a URL + return "", fmt.Errorf("%s: not a valid URL or hash", img) + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("%s: rkt only supports http or https URLs", img) + } + return fetchURL(img, ds) +} + func runFetch(args []string) (exit int) { if len(args) < 1 { fmt.Fprintf(os.Stderr, "fetch: Must provide at least one image\n") @@ -47,7 +82,7 @@ func runFetch(args []string) (exit int) { ds := cas.NewStore(globalFlags.Dir) for _, img := range args { - hash, err := fetchURL(img, ds) + hash, err := fetchImage(img, ds) if err != nil { fmt.Fprintf(os.Stderr, "%v", err) return 1 diff --git a/rkt/run.go b/rkt/run.go index 556089b..2c6ec84 100644 --- a/rkt/run.go +++ b/rkt/run.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "log" - "net/url" "os" "path/filepath" "strings" @@ -25,7 +24,7 @@ var ( Usage: "[--volume LABEL:SOURCE] IMAGE...", Description: `IMAGE should be a string referencing an image; either a hash, local file on disk, or URL. They will be checked in that order and the first match will be used.`, - Run: runRun, + Run: runRun, } ) @@ -36,6 +35,8 @@ func init() { flagVolumes = volumeMap{} } +// findImages will recognize a ACI hash and use that, import a local file, use +// discovery or download an ACI directly. func findImages(args []string, ds *cas.Store) (out []string, err error) { out = make([]string, len(args)) copy(out, args) @@ -59,15 +60,7 @@ func findImages(args []string, ds *cas.Store) (out []string, err error) { continue } - // download if it is a URL - u, err := url.Parse(img) - if err != nil { - return nil, fmt.Errorf("%s: not a valid URL or hash", img) - } - if u.Scheme != "http" && u.Scheme != "https" { - return nil, fmt.Errorf("%s: rkt only supports http or https URLs", img) - } - hash, err := fetchURL(img, ds) + hash, err := fetchImage(img, ds) if err != nil { return nil, err }