clearlinux/rkt
2
0
Fork 0
mirror of https://github.com/clearlinux/rkt.git synced 2026-09-06 05:41:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #102 from philips/hookup-discovery

Browse Source 
Hookup discovery
This commit is contained in:
Brandon Philips
2014-11-30 22:42:53 -08:00
parent e029b83c5e a03a63b609
commit 4cf487fe43
13 changed files with 151 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -6
View File
@@ -30,7 +30,7 @@ Keep in mind while running through the examples that rkt needs to be ran as root
Rocket uses content addressable storage (CAS) for storing an ACI on disk. In this example, the image is downloaded and added to the CAS.
```
$ rkt fetch https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace_validator_main.tar.gz
$ rkt fetch https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace-validator-main.tar.gz
sha256-f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8
```
@@ -47,11 +47,11 @@ $ find /var/lib/rkt/cas/object/
Per the App Container [spec][spec] the sha256 is of the tarball, which is reproducable with other tools:
```
$ wget https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace_validator_main.tar.gz
$ wget https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace-validator-main.tar.gz
...
$ gunzip ace_validator_main.tar.gz
$ sha256sum ace_validator_main.tar
f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ace_validator_main.tar
$ gunzip ace-validator-main.tar.gz
$ sha256sum ace-validator-main.tar
f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ace-validator-main.tar
```
### Launching an ACI
@@ -59,7 +59,7 @@ f59cb373728bd0f73674cc0b50286e56ba15bdd15a9e4ce8ccca18d0b6034ce8 ace_validator_
To run an ACI, you can either use the sha256 hash, or the URL which you downloaded it from:
```
$ rkt run https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace_validator_main.tar.gz
$ rkt run https://storage-download.googleapis.com/users.developer.core-os.net/philips/validator/ace-validator-main.tar.gz
```
rkt will do the appropriate etag checking on the URL to make sure it has the most up to date version of the image.
app-container/SPEC.md
+2 -2
View File
@@ -259,8 +259,8 @@ Then inspect the HTML returned for meta tags that have the following format:
Some examples for different schemes and URLs:
```
<meta name="ac-discovery" content="example.com https://storage.example.com/{os}/{arch}/{name}.{ext}?torrent">
<meta name="ac-discovery" content="example.com hdfs://storage.example.com/{name}-{os}-{arch}.{ext}">
<meta name="ac-discovery" content="example.com https://storage.example.com/{os}/{arch}/{name}-{version}.{ext}?torrent">
<meta name="ac-discovery" content="example.com hdfs://storage.example.com/{name}-{version}-{os}-{arch}.{ext}">
<meta name="ac-discovery-keys" content="example.com https://example.com/pubkeys.gpg">
```
app-container/ace/app_manifest_main.json
+3 -3
View File
@@ -5,19 +5,19 @@
"os": "linux",
"arch": "amd64",
"exec": [
"/ace_validator", "main"
"/ace-validator", "main"
],
"eventHandlers": [
{
"name": "pre-start",
"exec": [
"/ace_validator", "prestart"
"/ace-validator", "prestart"
]
},
{
"name": "post-stop",
"exec": [
"/ace_validator", "poststop"
"/ace-validator", "poststop"
]
}
],
app-container/ace/app_manifest_sidekick.json
+1 -1
View File
@@ -5,7 +5,7 @@
"os": "linux",
"arch": "amd64",
"exec": [
"/ace_validator", "sidekick"
"/ace-validator", "sidekick"
],
"user": "0",
"group": "0",
app-container/ace/build_aci
+11 -11
View File
@@ -13,25 +13,25 @@ fi
for typ in main sidekick; do
layoutdir="bin/ace_${typ}_layout"
mkdir -p ${layoutdir}/rootfs
cp bin/ace_validator ${layoutdir}/rootfs/
cp bin/ace-validator ${layoutdir}/rootfs/
cp ${PREFIX}/app_manifest_${typ}.json ${layoutdir}/app
# now build the tarball, and sign it
pushd ${layoutdir} >/dev/null
# Set a consistent timestamp so we get a consistent hash
# TODO(jonboulle): make this cleaner..
for path in rootfs rootfs/ace_validator; do
for path in rootfs rootfs/ace-validator; do
touch -a -m -d 1415660606 ${path}
done
../actool build --overwrite --app-manifest app rootfs/ ../ace_validator_${typ}.aci
HASH=sha256-$(sha256sum ../ace_validator_${typ}.aci|awk '{print $1}')
gzip -f ../ace_validator_${typ}.aci
mv ../ace_validator_${typ}.aci.gz ../ace_validator_${typ}.aci
gpg --cipher-algo AES256 --output ace_validator_${typ}.sig --detach-sig ../ace_validator_${typ}.aci
mv ace_validator_${typ}.sig ../
../actool build --overwrite --app-manifest app rootfs/ ../ace-validator-${typ}.aci
HASH=sha256-$(sha256sum ../ace-validator-${typ}.aci|awk '{print $1}')
gzip -f ../ace-validator-${typ}.aci
mv ../ace-validator-${typ}.aci.gz ../ace-validator-${typ}.aci
gpg --cipher-algo AES256 --output ace-validator-${typ}.sig --detach-sig ../ace-validator-${typ}.aci
mv ace-validator-${typ}.sig ../
popd >/dev/null
echo "Wrote ${typ} layout to ${layoutdir}"
echo "Wrote unsigned ${typ} ACI bin/ace_validator_${typ}.aci"
ln -s ${PWD}/bin/ace_validator_${typ}.aci bin/${HASH}
echo "Wrote unsigned ${typ} ACI bin/ace-validator-${typ}.aci"
ln -s ${PWD}/bin/ace-validator-${typ}.aci bin/${HASH}
echo "Wrote ${typ} layout hash bin/${HASH}"
echo "Wrote ${typ} ACI signature bin/ace_validator_${typ}.sig"
echo "Wrote ${typ} ACI signature bin/ace-validator-${typ}.sig"
done
app-container/actool/discover.go
+2 -2
View File
@@ -30,12 +30,12 @@ func runDiscover(args []string) (exit int) {
q := globalFlags.Quiet
for _, name := range args {
labels, err := appFromString(name)
app, err := discovery.NewAppFromString(name)
if err != nil {
stderr(q, "%s: %s", name, err)
return 1
}
eps, err := discovery.DiscoverEndpoints(labels["name"], labels["ver"], labels["os"], labels["amd64"], transportFlags.Insecure)
eps, err := discovery.DiscoverEndpoints(*app, transportFlags.Insecure)
if err != nil {
stderr(q, "error fetching %s: %s", name, err)
app-container/actool/fetch.go
-44
View File
@@ -1,44 +0,0 @@
package main
import (
"fmt"
"os"
"strings"
"github.com/coreos/rocket/app-container/discovery"
)
var (
cmdFetch = &Command{
Name: "fetch",
Description: "Discover and download an app container image",
Summary: "Discover, download and store on disk the app container image for one or more apps",
Run: runFetch,
}
)
func runFetch(args []string) (exit int) {
if len(args) < 1 {
fmt.Fprintf(os.Stderr, "discover: at least one name required")
}
q := globalFlags.Quiet
for _, name := range args {
labels, err := appFromString(name)
if err != nil {
stderr(q, "%s: %s", name, err)
return 1
}
eps, err := discovery.DiscoverEndpoints(labels["name"], labels["ver"], labels["os"], labels["arch"], transportFlags.Insecure)
if err != nil {
stderr(q, "error fetching %s: %s", name, err)
return 1
}
// TODO(philips): store the images..
fmt.Println(strings.Join(eps.Sig, ","))
fmt.Println(strings.Join(eps.ACI, ","))
fmt.Println(strings.Join(eps.Keys, ","))
}
return
}
app-container/actool/util.go
-45
View File
@@ -1,45 +0,0 @@
package main
import (
"fmt"
"net/url"
"runtime"
"strings"
)
const (
defaultVersion = "latest"
defaultOS = runtime.GOOS
defaultArch = runtime.GOARCH
)
// appFromString takes a command line app parameter and returns a map of labels.
//
// Example app parameters:
// example.com/reduce-worker:1.0.0
// example.com/reduce-worker,channel=alpha,label=value
func appFromString(app string) (out map[string]string, err error) {
out = make(map[string]string, 0)
app = strings.Replace(app, ":", ",ver=", -1)
app = "name=" + app
v, err := url.ParseQuery(strings.Replace(app, ",", "&", -1))
if err != nil {
return nil, err
}
for key, val := range v {
if len(val) > 1 {
return nil, fmt.Errorf("label %s with multiple values %q", key, val)
}
out[key] = val[0]
}
if out["ver"] == "" {
out["ver"] = defaultVersion
}
if out["os"] == "" {
out["os"] = defaultOS
}
if out["arch"] == "" {
out["arch"] = defaultArch
}
return
}
app-container/discovery/discovery.go
+5 -5
View File
@@ -78,8 +78,8 @@ func renderTemplate(tpl string, kvs ...string) string {
return tpl
}
func DiscoverEndpoints(app, ver, os, arch string, insecure bool) (*Endpoints, error) {
_, body, err := httpsOrHTTP(app, insecure)
func DiscoverEndpoints(app App, insecure bool) (*Endpoints, error) {
_, body, err := httpsOrHTTP(app.Name.String(), insecure)
if err != nil {
return nil, err
}
@@ -87,13 +87,13 @@ func DiscoverEndpoints(app, ver, os, arch string, insecure bool) (*Endpoints, er
meta := extractACMeta(body)
appver := app + "-" + ver
tplVars := []string{"{os}", os, "{arch}", arch, "{app-ver}", appver}
tplVars := []string{"{os}", app.Labels["os"], "{arch}", app.Labels["arch"],
"{name}", app.Name.String(), "{version}", app.Labels["version"]}
de := &Endpoints{}
for _, m := range meta {
if !strings.HasPrefix(appver, m.prefix) {
if !strings.HasPrefix(app.Name.String(), m.prefix) {
continue
}
app-container/discovery/parse.go
+80
View File
@@ -0,0 +1,80 @@
package discovery
import (
"fmt"
"net/url"
"runtime"
"strings"
"github.com/coreos/rocket/app-container/schema/types"
)
const (
defaultVersion = "latest"
defaultOS = runtime.GOOS
defaultArch = runtime.GOARCH
)
type App struct {
Name types.ACName
Labels map[string]string
}
func NewApp(name string, labels map[string]string) (*App, error) {
if labels == nil {
labels = make(map[string]string, 0)
}
acn, err := types.NewACName(name)
if err != nil {
return nil, err
}
return &App{
Name: *acn,
Labels: labels,
}, nil
}
// NewAppFromString takes a command line app parameter and returns a map of labels.
//
// Example app parameters:
// example.com/reduce-worker:1.0.0
// example.com/reduce-worker,channel=alpha,label=value
func NewAppFromString(app string) (*App, error) {
var (
name string
labels map[string]string
)
app = strings.Replace(app, ":", ",version=", -1)
app = "name=" + app
v, err := url.ParseQuery(strings.Replace(app, ",", "&", -1))
if err != nil {
return nil, err
}
labels = make(map[string]string, 0)
for key, val := range v {
if len(val) > 1 {
return nil, fmt.Errorf("label %s with multiple values %q", key, val)
}
if key == "name" {
name = val[0]
continue
}
labels[key] = val[0]
}
if labels["version"] == "" {
labels["version"] = defaultVersion
}
if labels["os"] == "" {
labels["os"] = defaultOS
}
if labels["arch"] == "" {
labels["arch"] = defaultArch
}
a, err := NewApp(name, labels)
if err != nil {
return nil, err
}
return a, nil
}
build
+1 -1
View File
@@ -17,7 +17,7 @@ echo "Building actool..."
go build -o $GOBIN/actool ${REPO_PATH}/app-container/actool
echo "Building ACE validator..."
GOOS=linux CGO_ENABLED=0 go build -a -ldflags '-extldflags "-static"' -o $GOBIN/ace_validator ${REPO_PATH}/app-container/ace
GOOS=linux CGO_ENABLED=0 go build -a -ldflags '-extldflags "-static"' -o $GOBIN/ace-validator ${REPO_PATH}/app-container/ace
echo "Building init (stage1)..."
go build -o $GOBIN/init ${REPO_PATH}/stage1
rkt/fetch.go
+36 -1
View File
@@ -2,9 +2,11 @@ package main
import (
"fmt"
"net/url"
"os"
"path/filepath"
"github.com/coreos/rocket/app-container/discovery"
"github.com/coreos/rocket/cas"
)
@@ -33,6 +35,39 @@ func fetchURL(img string, ds *cas.Store) (string, error) {
return rem.Blob, nil
}
// fetchImage will take an image as either a URL or a name string and import it
// into the store if found.
func fetchImage(img string, ds *cas.Store) (string, error) {
// discover if it isn't a URL
u, err := url.Parse(img)
if err == nil && u.Scheme == "" {
app, err := discovery.NewAppFromString(img)
if globalFlags.Debug && err != nil {
fmt.Printf("discovery: %s\n", err)
}
if err == nil {
ep, err := discovery.DiscoverEndpoints(*app, true)
if err != nil {
return "", err
}
// TODO(philips): use all available mirrors
if globalFlags.Debug {
fmt.Printf("fetch: trying %v\n", ep.ACI)
}
img = ep.ACI[0]
u, err = url.Parse(img)
}
}
if err != nil { // download if it isn't a URL
return "", fmt.Errorf("%s: not a valid URL or hash", img)
}
if u.Scheme != "http" && u.Scheme != "https" {
return "", fmt.Errorf("%s: rkt only supports http or https URLs", img)
}
return fetchURL(img, ds)
}
func runFetch(args []string) (exit int) {
if len(args) < 1 {
fmt.Fprintf(os.Stderr, "fetch: Must provide at least one image\n")
@@ -47,7 +82,7 @@ func runFetch(args []string) (exit int) {
ds := cas.NewStore(globalFlags.Dir)
for _, img := range args {
hash, err := fetchURL(img, ds)
hash, err := fetchImage(img, ds)
if err != nil {
fmt.Fprintf(os.Stderr, "%v", err)
return 1
rkt/run.go
+4 -11
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
"path/filepath"
"strings"
@@ -25,7 +24,7 @@ var (
Usage: "[--volume LABEL:SOURCE] IMAGE...",
Description: `IMAGE should be a string referencing an image; either a hash, local file on disk, or URL.
They will be checked in that order and the first match will be used.`,
Run: runRun,
Run: runRun,
}
)
@@ -36,6 +35,8 @@ func init() {
flagVolumes = volumeMap{}
}
// findImages will recognize a ACI hash and use that, import a local file, use
// discovery or download an ACI directly.
func findImages(args []string, ds *cas.Store) (out []string, err error) {
out = make([]string, len(args))
copy(out, args)
@@ -59,15 +60,7 @@ func findImages(args []string, ds *cas.Store) (out []string, err error) {
continue
}
// download if it is a URL
u, err := url.Parse(img)
if err != nil {
return nil, fmt.Errorf("%s: not a valid URL or hash", img)
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("%s: rkt only supports http or https URLs", img)
}
hash, err := fetchURL(img, ds)
hash, err := fetchImage(img, ds)
if err != nil {
return nil, err
}