Integrate metadata service

- Move metadatasvc into rkt as a subcommand
- Add option to spawn metadatasvc from rkt run
- Register container with metadatasvc
- Removes anti-spoofing logic -- this belongs in the network plugin

Fixes #33

build

@@ -47,7 +47,3 @@ fi
 
 echo "Building rkt (stage0)..."
 go build -o $GOBIN/rkt ${REPO_PATH}/rkt
-
-echo "Building metadatasvc..."
-go build -o $GOBIN/metadatasvc ${REPO_PATH}/metadatasvc
-

metadata/metadatasvc.go

@@ -0,0 +1,17 @@
+package metadata
+
+import "fmt"
+
+const (
+	SvcIP      = "169.254.169.255"
+	SvcPubPort = 80
+	SvcPrvPort = 2375
+)
+
+func SvcPrvURL() string {
+	return fmt.Sprintf("http://127.0.0.1:%v", SvcPrvPort)
+}
+
+func SvcPubURL() string {
+	return fmt.Sprintf("http://%v:%v", SvcIP, SvcPubPort)
+}

networking/net-plugin.go

@@ -79,21 +79,21 @@ func (e *containerEnv) execNetPlugin(cmd string, n *Net, netns, args, ifName str
 	}
 
 	vars := [][2]string{
-		{ "RKT_NETPLUGIN_COMMAND", cmd },
-		{ "RKT_NETPLUGIN_CONTID", e.contID.String() },
-		{ "RKT_NETPLUGIN_NETNS", netns },
-		{ "RKT_NETPLUGIN_ARGS", args },
-		{ "RKT_NETPLUGIN_IFNAME", ifName },
-		{ "RKT_NETPLUGIN_NETNAME", n.Name },
-		{ "RKT_NETPLUGIN_NETCONF", n.Filename },
+		{"RKT_NETPLUGIN_COMMAND", cmd},
+		{"RKT_NETPLUGIN_CONTID", e.contID.String()},
+		{"RKT_NETPLUGIN_NETNS", netns},
+		{"RKT_NETPLUGIN_ARGS", args},
+		{"RKT_NETPLUGIN_IFNAME", ifName},
+		{"RKT_NETPLUGIN_NETNAME", n.Name},
+		{"RKT_NETPLUGIN_NETCONF", n.Filename},
 	}
 
 	stdout := &bytes.Buffer{}
 
 	c := exec.Cmd{
-		Path: pluginPath,
-		Args: []string{pluginPath},
-		Env: envVars(vars),
+		Path:   pluginPath,
+		Args:   []string{pluginPath},
+		Env:    envVars(vars),
 		Stdout: stdout,
 		Stderr: os.Stderr,
 	}

networking/net.go

@@ -33,6 +33,7 @@ type Net struct {
 
 // Absolute path where users place their net configs
 const UserNetPath = "/etc/rkt/net.d"
+
 // Default net path relative to stage1 root
 const DefaultNetPath = "etc/rkt/net.d/99-default.conf"
 

networking/networking.go

@@ -43,8 +43,8 @@ type activeNet struct {
 // describing the environment in which the container
 // is running in
 type containerEnv struct {
-	rktRoot    string
-	contID     types.UUID
+	rktRoot string
+	contID  types.UUID
 }
 
 // Networking describes the networking details of a container.
@@ -66,7 +66,7 @@ func Setup(rktRoot string, contID types.UUID) (*Networking, error) {
 	n := Networking{
 		containerEnv: containerEnv{
 			rktRoot: rktRoot,
-			contID: contID,
+			contID:  contID,
 		},
 	}
 

networking/plugins/veth/veth.go

@@ -125,7 +125,7 @@ func main() {
 	cmd := os.Getenv("RKT_NETPLUGIN_COMMAND")
 	contID := os.Getenv("RKT_NETPLUGIN_CONTID")
 	netns := os.Getenv("RKT_NETPLUGIN_NETNS")
-	args :=	os.Getenv("RKT_NETPLUGIN_ARGS")
+	args := os.Getenv("RKT_NETPLUGIN_ARGS")
 	ifName := os.Getenv("RKT_NETPLUGIN_IFNAME")
 	netConf := os.Getenv("RKT_NETPLUGIN_NETCONF")
 

rkt/metadatasvc.go

@@ -23,55 +23,69 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	"github.com/appc/spec/schema"
 	"github.com/appc/spec/schema/types"
 	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/gorilla/mux"
+
+	"github.com/coreos/rocket/metadata"
 )
 
-type metadata struct {
+var (
+	cmdMetadataSvc = &Command{
+		Name:    "metadatasvc",
+		Summary: "Run metadata service",
+		Usage:   "[--src-addr CIDR] [--listen-port PORT] [--no-idle]",
+		Run:     runMetadataSvc,
+	}
+)
+
+type container struct {
 	manifest schema.ContainerRuntimeManifest
 	apps     map[string]*schema.ImageManifest
+	ip       string
 }
 
 var (
-	metadataByIP  = make(map[string]*metadata)
-	metadataByUID = make(map[types.UUID]*metadata)
-	hmacKey       [sha256.Size]byte
+	containerByIP  = make(map[string]*container)
+	containerByUID = make(map[types.UUID]*container)
+	hmacKey        [sha256.Size]byte
+
+	flagListenPort int
+	flagSrcAddrs   string
+	flagNoIdle     bool
+
+	exitCh chan bool
 )
 
 const (
-	myPort   = "4444"
-	metaIP   = "169.254.169.255"
-	metaPort = "80"
+	listenFdsStart = 3
 )
 
-func setupIPTables() error {
+func init() {
+	commands = append(commands, cmdMetadataSvc)
+	cmdMetadataSvc.Flags.StringVar(&flagSrcAddrs, "src-addr", "0.0.0.0/0", "source address/range for iptables")
+	cmdMetadataSvc.Flags.IntVar(&flagListenPort, "listen-port", metadata.SvcPrvPort, "listen port")
+	cmdMetadataSvc.Flags.BoolVar(&flagNoIdle, "no-idle", false, "exit when last container is unregistered")
+}
+
+func modifyIPTables(action, port string) error {
 	return exec.Command(
 		"iptables",
 		"-t", "nat",
-		"-A", "PREROUTING",
+		action, "PREROUTING",
 		"-p", "tcp",
-		"-d", metaIP,
-		"--dport", metaPort,
+		"-d", metadata.SvcIP,
+		"--dport", strconv.Itoa(metadata.SvcPubPort),
 		"-j", "REDIRECT",
-		"--to-port", myPort,
-	).Run()
-}
-
-func antiSpoof(brPort, ipAddr string) error {
-	return exec.Command(
-		"ebtables",
-		"-t", "filter",
-		"-I", "INPUT",
-		"-i", brPort,
-		"-p", "IPV4",
-		"!", "--ip-source", ipAddr,
-		"-j", "DROP",
+		"--to-port", port,
 	).Run()
 }
 
@@ -84,51 +98,70 @@ func queryValue(u *url.URL, key string) string {
 }
 
 func handleRegisterContainer(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
 	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-	if _, ok := metadataByIP[remoteIP]; ok {
+	if _, ok := containerByIP[remoteIP]; ok {
 		// not allowed from container IP
 		w.WriteHeader(http.StatusForbidden)
 		return
 	}
 
-	containerIP := queryValue(r.URL, "container_ip")
+	containerIP := queryValue(r.URL, "ip")
 	if containerIP == "" {
 		w.WriteHeader(http.StatusBadRequest)
-		fmt.Print(w, "container_ip missing")
-		return
-	}
-	containerBrPort := queryValue(r.URL, "container_brport")
-	if containerBrPort == "" {
-		w.WriteHeader(http.StatusBadRequest)
-		fmt.Print(w, "container_brport missing")
+		fmt.Fprint(w, "ip missing")
 		return
 	}
 
-	m := &metadata{
+	c := &container{
 		apps: make(map[string]*schema.ImageManifest),
+		ip:   containerIP,
 	}
 
-	if err := json.NewDecoder(r.Body).Decode(&m.manifest); err != nil {
+	if err := json.NewDecoder(r.Body).Decode(&c.manifest); err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		fmt.Fprintf(w, "JSON-decoding failed: %v", err)
 		return
 	}
 
-	if err := antiSpoof(containerBrPort, containerIP); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		fmt.Fprintf(w, "failed to set anti-spoofing: %v", err)
-		return
-	}
-
-	metadataByIP[containerIP] = m
-	metadataByUID[m.manifest.UUID] = m
+	containerByIP[containerIP] = c
+	containerByUID[c.manifest.UUID] = c
 
 	w.WriteHeader(http.StatusOK)
 }
 
+func handleUnregisterContainer(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
+	uid, err := types.NewUUID(mux.Vars(r)["uid"])
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		fmt.Fprintf(w, "UUID is missing or malformed: %v", err)
+		return
+	}
+
+	c, ok := containerByUID[*uid]
+	if !ok {
+		w.WriteHeader(http.StatusNotFound)
+		fmt.Fprint(w, "Container with given UUID not found")
+		return
+	}
+
+	delete(containerByUID, *uid)
+	delete(containerByIP, c.ip)
+	w.WriteHeader(http.StatusOK)
+
+	if flagNoIdle && len(containerByUID) == 0 {
+		exitCh <- true
+	}
+}
+
 func handleRegisterApp(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
 	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-	if _, ok := metadataByIP[remoteIP]; ok {
+	if _, ok := containerByIP[remoteIP]; ok {
 		// not allowed from container IP
 		w.WriteHeader(http.StatusForbidden)
 		return
@@ -141,7 +174,7 @@ func handleRegisterApp(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	m, ok := metadataByUID[*uid]
+	c, ok := containerByUID[*uid]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		fmt.Fprint(w, "Container with given UUID not found")
@@ -157,31 +190,31 @@ func handleRegisterApp(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	m.apps[an] = app
+	c.apps[an] = app
 
 	w.WriteHeader(http.StatusOK)
 }
 
-func containerGet(h func(w http.ResponseWriter, r *http.Request, m *metadata)) http.HandlerFunc {
+func containerGet(h func(w http.ResponseWriter, r *http.Request, c *container)) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-		m, ok := metadataByIP[remoteIP]
+		c, ok := containerByIP[remoteIP]
 		if !ok {
 			w.WriteHeader(http.StatusNotFound)
-			fmt.Fprintf(w, "metadata by remoteIP (%v) not found", remoteIP)
+			fmt.Fprintf(w, "container by remoteIP (%v) not found", remoteIP)
 			return
 		}
 
-		h(w, r, m)
+		h(w, r, c)
 	}
 }
 
-func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, _ *schema.ImageManifest)) http.HandlerFunc {
-	return containerGet(func(w http.ResponseWriter, r *http.Request, m *metadata) {
+func appGet(h func(w http.ResponseWriter, r *http.Request, c *container, _ *schema.ImageManifest)) http.HandlerFunc {
+	return containerGet(func(w http.ResponseWriter, r *http.Request, c *container) {
 		appname := mux.Vars(r)["app"]
 
-		if im, ok := m.apps[appname]; ok {
-			h(w, r, m, im)
+		if im, ok := c.apps[appname]; ok {
+			h(w, r, c, im)
 		} else {
 			w.WriteHeader(http.StatusNotFound)
 			fmt.Fprintf(w, "App (%v) not found", appname)
@@ -189,16 +222,20 @@ func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, _ *schem
 	})
 }
 
-func handleContainerAnnotations(w http.ResponseWriter, r *http.Request, m *metadata) {
+func handleContainerAnnotations(w http.ResponseWriter, r *http.Request, c *container) {
+	defer r.Body.Close()
+
 	w.Header().Add("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
 
-	for k := range m.manifest.Annotations {
+	for k := range c.manifest.Annotations {
 		fmt.Fprintln(w, k)
 	}
 }
 
-func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metadata) {
+func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, c *container) {
+	defer r.Body.Close()
+
 	k, err := types.NewACName(mux.Vars(r)["name"])
 	if err != nil {
 		w.WriteHeader(http.StatusNotFound)
@@ -206,7 +243,7 @@ func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metada
 		return
 	}
 
-	v, ok := m.manifest.Annotations.Get(k.String())
+	v, ok := c.manifest.Annotations.Get(k.String())
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		fmt.Fprintf(w, "Container annotation (%v) not found", k)
@@ -218,17 +255,21 @@ func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metada
 	w.Write([]byte(v))
 }
 
-func handleContainerManifest(w http.ResponseWriter, r *http.Request, m *metadata) {
+func handleContainerManifest(w http.ResponseWriter, r *http.Request, c *container) {
+	defer r.Body.Close()
+
 	w.Header().Add("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 
-	if err := json.NewEncoder(w).Encode(m.manifest); err != nil {
-		fmt.Println(err)
+	if err := json.NewEncoder(w).Encode(c.manifest); err != nil {
+		log.Print(err)
 	}
 }
 
-func handleContainerUID(w http.ResponseWriter, r *http.Request, m *metadata) {
-	uid := m.manifest.UUID.String()
+func handleContainerUID(w http.ResponseWriter, r *http.Request, c *container) {
+	defer r.Body.Close()
+
+	uid := c.manifest.UUID.String()
 
 	w.Header().Add("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
@@ -251,16 +292,20 @@ func mergeAppAnnotations(im *schema.ImageManifest, cm *schema.ContainerRuntimeMa
 	return merged
 }
 
-func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
+func handleAppAnnotations(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) {
+	defer r.Body.Close()
+
 	w.Header().Add("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
 
-	for _, annot := range mergeAppAnnotations(im, &m.manifest) {
+	for _, annot := range mergeAppAnnotations(im, &c.manifest) {
 		fmt.Fprintln(w, string(annot.Name))
 	}
 }
 
-func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
+func handleAppAnnotation(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) {
+	defer r.Body.Close()
+
 	k, err := types.NewACName(mux.Vars(r)["name"])
 	if err != nil {
 		w.WriteHeader(http.StatusNotFound)
@@ -268,7 +313,7 @@ func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im
 		return
 	}
 
-	merged := mergeAppAnnotations(im, &m.manifest)
+	merged := mergeAppAnnotations(im, &c.manifest)
 
 	v, ok := merged.Get(k.String())
 	if !ok {
@@ -282,19 +327,23 @@ func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im
 	w.Write([]byte(v))
 }
 
-func handleImageManifest(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
+func handleImageManifest(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) {
+	defer r.Body.Close()
+
 	w.Header().Add("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 
 	if err := json.NewEncoder(w).Encode(*im); err != nil {
-		fmt.Println(err)
+		log.Print(err)
 	}
 }
 
-func handleAppID(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
+func handleAppID(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) {
+	defer r.Body.Close()
+
 	w.Header().Add("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
-	a := m.manifest.Apps.Get(im.Name)
+	a := c.manifest.Apps.Get(im.Name)
 	if a == nil {
 		panic("could not find app in manifest!")
 	}
@@ -317,8 +366,10 @@ func digest(r io.Reader) ([]byte, error) {
 }
 
 func handleContainerSign(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
 	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-	m, ok := metadataByIP[remoteIP]
+	c, ok := containerByIP[remoteIP]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		fmt.Fprintf(w, "Metadata by remoteIP (%v) not found", remoteIP)
@@ -335,7 +386,7 @@ func handleContainerSign(w http.ResponseWriter, r *http.Request) {
 
 	// HMAC(UID:digest)
 	h := hmac.New(sha256.New, hmacKey[:])
-	h.Write(m.manifest.UUID[:])
+	h.Write(c.manifest.UUID[:])
 	h.Write(d)
 
 	// Send back digest:HMAC as the signature
@@ -348,6 +399,8 @@ func handleContainerSign(w http.ResponseWriter, r *http.Request) {
 }
 
 func handleContainerVerify(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+
 	uid, err := types.NewUUID(r.FormValue("uid"))
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
@@ -398,21 +451,14 @@ func logReq(h func(w http.ResponseWriter, r *http.Request)) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		resp := &httpResp{w, 0}
 		h(resp, r)
-		fmt.Printf("%v %v - %v\n", r.Method, r.RequestURI, resp.status)
+		log.Printf("%v %v - %v", r.Method, r.RequestURI, resp.status)
 	}
 }
 
-func main() {
-	if err := setupIPTables(); err != nil {
-		log.Fatal(err)
-	}
-
-	if err := initCrypto(); err != nil {
-		log.Fatal(err)
-	}
-
+func makeHandlers() http.Handler {
 	r := mux.NewRouter()
 	r.HandleFunc("/containers/", logReq(handleRegisterContainer)).Methods("POST")
+	r.HandleFunc("/containers/{uid}", logReq(handleUnregisterContainer)).Methods("DELETE")
 	r.HandleFunc("/containers/{uid}/{app:.*}", logReq(handleRegisterApp)).Methods("PUT")
 
 	acRtr := r.Headers("Metadata-Flavor", "AppContainer").
@@ -433,5 +479,83 @@ func main() {
 	acRtr.HandleFunc("/container/hmac/sign", logReq(handleContainerSign)).Methods("POST")
 	acRtr.HandleFunc("/container/hmac/verify", logReq(handleContainerVerify)).Methods("POST")
 
-	log.Fatal(http.ListenAndServe(":4444", r))
+	return r
+}
+
+func getListener() (net.Listener, error) {
+	s := os.Getenv("LISTEN_FDS")
+	if s != "" {
+		// socket activated
+		lfds, err := strconv.ParseInt(s, 10, 16)
+		if err != nil {
+			return nil, fmt.Errorf("Error parsing LISTEN_FDS env var: %v", err)
+		}
+		if lfds < 1 {
+			return nil, fmt.Errorf("LISTEN_FDS < 1")
+		}
+
+		return net.FileListener(os.NewFile(uintptr(listenFdsStart), "listen"))
+	} else {
+		return net.Listen("tcp4", fmt.Sprintf(":%v", flagListenPort))
+	}
+}
+
+func cleanup(port string) {
+	if err := modifyIPTables("-D", port); err != nil {
+		log.Printf("Error cleaning up iptables: %v", err)
+	}
+}
+
+func runMetadataSvc(args []string) (exit int) {
+	log.Print("Metadatasvc starting...")
+
+	l, err := getListener()
+	if err != nil {
+		log.Printf("Error getting listener: %v", err)
+		return
+	}
+
+	initCrypto()
+
+	port := strings.Split(l.Addr().String(), ":")[1]
+
+	if flagNoIdle {
+		// TODO(eyakubovich): this is very racy
+		// It's possible for last container to get unregistered
+		// and svc gets flagged to shutdown. Then another container
+		// starts to launch, sees that port is in use and doesn't
+		// start metadata svc only for this one to exit a moment later.
+		// However, --no-idle is meant for demos and having a single
+		// container spawn up (via --spawn-metadata-svc). The design
+		// of metadata svc is also likely to change as we convert it
+		// to be backed by persistent storage.
+		exitCh = make(chan bool, 1)
+		// wait for signal and exit
+		go func() {
+			<-exitCh
+			cleanup(port)
+			os.Exit(0)
+		}()
+	}
+
+	if err := modifyIPTables("-A", port); err != nil {
+		log.Printf("Error setting up iptables: %v", err)
+		return 1
+	}
+
+	srv := http.Server{
+		Handler: makeHandlers(),
+	}
+
+	log.Print("Metadatasvc running...")
+
+	if err = srv.Serve(l); err != nil {
+		log.Printf("Error serving HTTP: %v", err)
+		exit = 1
+	}
+
+	cleanup(port)
+	log.Print("Metadatasvc exiting...")
+
+	return
 }

rkt/rkt.go

@@ -17,6 +17,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"text/tabwriter"
@@ -93,6 +94,11 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Run '%v help' for usage.\n", cliName)
 		os.Exit(2)
 	}
+
+	if globalFlags.Debug {
+		log.SetOutput(os.Stderr)
+	}
+
 	os.Exit(cmd.Run(cmd.Flags.Args()))
 }
 

rkt/run.go

@@ -31,11 +31,12 @@ import (
 )
 
 var (
-	flagStage1Init   string
-	flagStage1Rootfs string
-	flagVolumes      volumeMap
-	flagPrivateNet   bool
-	cmdRun           = &Command{
+	flagStage1Init       string
+	flagStage1Rootfs     string
+	flagVolumes          volumeMap
+	flagPrivateNet       bool
+	flagSpawnMetadataSvc bool
+	cmdRun               = &Command{
 		Name:    "run",
 		Summary: "Run image(s) in an application container in rocket",
 		Usage:   "[--volume LABEL:SOURCE] IMAGE...",
@@ -51,6 +52,7 @@ func init() {
 	cmdRun.Flags.StringVar(&flagStage1Rootfs, "stage1-rootfs", "", "path to stage1 rootfs tarball override")
 	cmdRun.Flags.Var(&flagVolumes, "volume", "volumes to mount into the shared container environment")
 	cmdRun.Flags.BoolVar(&flagPrivateNet, "private-net", false, "give container a private network")
+	cmdRun.Flags.BoolVar(&flagSpawnMetadataSvc, "spawn-metadata-svc", false, "launch metadata svc if not running")
 	flagVolumes = volumeMap{}
 }
 
@@ -131,14 +133,15 @@ func runRun(args []string) (exit int) {
 	}
 
 	cfg := stage0.Config{
-		Store:         ds,
-		ContainersDir: containersDir(),
-		Debug:         globalFlags.Debug,
-		Stage1Init:    flagStage1Init,
-		Stage1Rootfs:  flagStage1Rootfs,
-		Images:        imgs,
-		Volumes:       flagVolumes,
-		PrivateNet:    flagPrivateNet,
+		Store:            ds,
+		ContainersDir:    containersDir(),
+		Debug:            globalFlags.Debug,
+		Stage1Init:       flagStage1Init,
+		Stage1Rootfs:     flagStage1Rootfs,
+		Images:           imgs,
+		Volumes:          flagVolumes,
+		PrivateNet:       flagPrivateNet,
+		SpawnMetadataSvc: flagSpawnMetadataSvc,
 	}
 	cdir, err := stage0.Setup(cfg)
 	if err != nil {

stage0/run.go

@@ -64,9 +64,10 @@ type Config struct {
 	Stage1Rootfs  string     // compressed bundle containing a rootfs for stage1
 	Debug         bool
 	// TODO(jonboulle): These images are partially-populated hashes, this should be clarified.
-	Images     []types.Hash      // application images
-	Volumes    map[string]string // map of volumes that rocket can provide to applications
-	PrivateNet bool              // container should have its own network stack
+	Images           []types.Hash      // application images
+	Volumes          map[string]string // map of volumes that rocket can provide to applications
+	PrivateNet       bool              // container should have its own network stack
+	SpawnMetadataSvc bool              // launch metadata service
 }
 
 func init() {
@@ -76,10 +77,6 @@ func init() {
 // Setup sets up a filesystem for a container based on the given config.
 // The directory containing the filesystem is returned, and any error encountered.
 func Setup(cfg Config) (string, error) {
-	if cfg.Debug {
-		log.SetOutput(os.Stderr)
-	}
-
 	cuuid, err := types.NewUUID(uuid.New())
 	if err != nil {
 		return "", fmt.Errorf("error creating UID: %v", err)
@@ -207,6 +204,17 @@ func Run(cfg Config, dir string) {
 	if cfg.Debug {
 		args = append(args, "--debug")
 	}
+	if cfg.SpawnMetadataSvc {
+		rktExe, err := os.Readlink("/proc/self/exe")
+		if err != nil {
+			log.Fatalf("failed to readlink /proc/self/exe: %v", err)
+		}
+		dbgFlag := ""
+		if cfg.Debug {
+			dbgFlag = " --debug"
+		}
+		args = append(args, fmt.Sprintf("--metadata-svc=%s%s metadatasvc --no-idle", rktExe, dbgFlag))
+	}
 	if cfg.PrivateNet {
 		args = append(args, "--private-net")
 	}

stage1/init/container.go

@@ -34,9 +34,11 @@ import (
 
 // Container encapsulates a ContainerRuntimeManifest and ImageManifests
 type Container struct {
-	Root     string // root directory where the container will be located
-	Manifest *schema.ContainerRuntimeManifest
-	Apps     map[string]*schema.ImageManifest
+	Root           string // root directory where the container will be located
+	Manifest       *schema.ContainerRuntimeManifest
+	Apps           map[string]*schema.ImageManifest
+	MetadataSvcURL string
+	Networks       []string
 }
 
 // LoadContainer loads a Container Runtime Manifest (as prepared by stage0) and
@@ -147,6 +149,8 @@ func (c *Container) appToSystemd(am *schema.ImageManifest, id types.Hash) error
 
 	env := app.Environment
 	env["AC_APP_NAME"] = name
+	env["AC_METADATA_URL"] = c.MetadataSvcURL
+
 	for ek, ev := range env {
 		ee := fmt.Sprintf(`"%s=%s"`, ek, ev)
 		opts = append(opts, newUnitOption("Service", "Environment", ee))
@@ -267,10 +271,6 @@ func (c *Container) appToNspawnArgs(am *schema.ImageManifest, id types.Hash) ([]
 
 	for _, i := range am.App.Isolators {
 		switch i.Name {
-		case "private-network":
-			if i.Val == "true" {
-				args = append(args, "--private-network")
-			}
 		case "capabilities/bounding-set":
 			capList := strings.Join(strings.Split(i.Val, " "), ",")
 			args = append(args, "--capability="+capList)

stage1/init/init.go

@@ -22,12 +22,16 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"syscall"
 
+	"github.com/coreos/rocket/metadata"
 	"github.com/coreos/rocket/networking"
 	"github.com/coreos/rocket/path"
 )
@@ -76,13 +80,15 @@ func mirrorLocalZoneInfo(root string) {
 }
 
 var (
-	debug   bool
-	privNet bool
+	debug       bool
+	metadataSvc string
+	privNet     bool
 )
 
 func init() {
 	flag.BoolVar(&debug, "debug", false, "Run in debug mode")
 	flag.BoolVar(&privNet, "private-net", false, "Setup private network (WIP!)")
+	flag.StringVar(&metadataSvc, "metadata-svc", "", "Launch specified metadata svc")
 
 	// this ensures that main runs only on main thread (thread group leader).
 	// since namespace ops (unshare, setns) are done for a single thread, we
@@ -99,6 +105,7 @@ func stage1() int {
 	}
 
 	mirrorLocalZoneInfo(c.Root)
+	c.MetadataSvcURL = metadata.SvcPubURL()
 
 	if err = c.ContainerToSystemd(); err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to configure systemd: %v\n", err)
@@ -135,6 +142,13 @@ func stage1() int {
 	env = append(env, "LD_PRELOAD="+filepath.Join(path.Stage1RootfsPath(c.Root), "fakesdboot.so"))
 	env = append(env, "LD_LIBRARY_PATH="+filepath.Join(path.Stage1RootfsPath(c.Root), "usr/lib"))
 
+	if metadataSvc != "" {
+		if err = launchMetadataSvc(); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to launch metadata svc: %v\n", err)
+			return 6
+		}
+	}
+
 	if privNet {
 		// careful not to make another local err variable.
 		// cmd.Run sets the one from parent scope
@@ -151,6 +165,12 @@ func stage1() int {
 			return 6
 		}
 
+		if err = registerContainer(c, n.MetadataIP); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to register container: %v\n", err)
+			return 6
+		}
+		defer unregisterContainer(c)
+
 		cmd := exec.Cmd{
 			Path:   args[0],
 			Args:   args,
@@ -172,6 +192,41 @@ func stage1() int {
 	return 0
 }
 
+func launchMetadataSvc() error {
+	log.Print("Launching metadatasvc: ", metadataSvc)
+
+	// use socket activation protocol to avoid race-condition of
+	// service becoming ready
+	// TODO(eyakubovich): remove hard-coded port
+	l, err := net.ListenTCP("tcp4", &net.TCPAddr{Port: metadata.SvcPrvPort})
+	if err != nil {
+		if err.(*net.OpError).Err.(*os.SyscallError).Err == syscall.EADDRINUSE {
+			// assume metadatasvc is already running
+			return nil
+		}
+		return err
+	}
+
+	defer l.Close()
+
+	lf, err := l.File()
+	if err != nil {
+		return err
+	}
+
+	// parse metadataSvc into exe and args
+	args := strings.Split(metadataSvc, " ")
+	cmd := exec.Cmd{
+		Path:       args[0],
+		Args:       args,
+		Env:        append(os.Environ(), "LISTEN_FDS=1"),
+		ExtraFiles: []*os.File{lf},
+		Stdout:     os.Stdout,
+		Stderr:     os.Stderr,
+	}
+	return cmd.Start()
+}
+
 func main() {
 	flag.Parse()
 	// move code into stage1() helper so defered fns get run

stage1/init/registration.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path"
+
+	"github.com/coreos/rocket/metadata"
+	rktpath "github.com/coreos/rocket/path"
+)
+
+func registerContainer(c *Container, ip net.IP) error {
+	cmf, err := os.Open(rktpath.ContainerManifestPath(c.Root))
+	if err != nil {
+		return fmt.Errorf("failed opening runtime manifest: %v", err)
+	}
+	defer cmf.Close()
+
+	pth := fmt.Sprintf("/containers/?ip=%v", ip.To4().String())
+	if err := httpRequest("POST", pth, cmf); err != nil {
+		return fmt.Errorf("failed to register container with metadata svc: %v", err)
+	}
+
+	uid := c.Manifest.UUID.String()
+	for _, app := range c.Manifest.Apps {
+		ampath := rktpath.ImageManifestPath(c.Root, app.ImageID)
+		amf, err := os.Open(ampath)
+		if err != nil {
+			fmt.Errorf("failed reading app manifest %q: %v", ampath, err)
+		}
+		defer amf.Close()
+
+		if err := registerApp(uid, app.Name.String(), amf); err != nil {
+			fmt.Errorf("failed to register app with metadata svc: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func unregisterContainer(c *Container) error {
+	pth := path.Join("/containers", c.Manifest.UUID.String())
+	return httpRequest("DELETE", pth, nil)
+}
+
+func registerApp(uuid, app string, r io.Reader) error {
+	pth := path.Join("/containers", uuid, app)
+	return httpRequest("PUT", pth, r)
+}
+
+func httpRequest(method, pth string, body io.Reader) error {
+	uri := metadata.SvcPrvURL() + pth
+	req, err := http.NewRequest(method, uri, body)
+	if err != nil {
+		return err
+	}
+
+	cli := http.Client{}
+
+	resp, err := cli.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("%v %v returned %v", method, pth, resp.StatusCode)
+	}
+
+	return nil
+}

test

@@ -15,7 +15,7 @@ COVER=${COVER:-"-cover"}
 source ./build
 
 TESTABLE_AND_FORMATTABLE="cas pkg/keystore pkg/lock pkg/tar rkt stage1/init"
-FORMATTABLE="$TESTABLE_AND_FORMATTABLE metadatasvc path pkg/io pkg/proc stage0/run.go version"
+FORMATTABLE="$TESTABLE_AND_FORMATTABLE networking path pkg/io pkg/proc stage0/run.go version"
 
 # user has not provided PKG override
 if [ -z "$PKG" ]; then