From 25ecad81623917337435ab047aa2e08d59166384 Mon Sep 17 00:00:00 2001 From: Eugene Yakubovich Date: Fri, 23 Jan 2015 13:25:46 -0800 Subject: [PATCH] Integrate metadata service - Move metadatasvc into rkt as a subcommand - Add option to spawn metadatasvc from rkt run - Register container with metadatasvc - Removes anti-spoofing logic -- this belongs in the network plugin Fixes #33 --- build | 4 - metadata/metadatasvc.go | 17 ++ networking/net-plugin.go | 20 +- networking/net.go | 1 + networking/networking.go | 6 +- networking/plugins/veth/veth.go | 2 +- metadatasvc/main.go => rkt/metadatasvc.go | 292 +++++++++++++++------- rkt/rkt.go | 6 + rkt/run.go | 29 ++- stage0/run.go | 22 +- stage1/init/container.go | 14 +- stage1/init/init.go | 59 ++++- stage1/init/registration.go | 74 ++++++ test | 2 +- 14 files changed, 416 insertions(+), 132 deletions(-) create mode 100644 metadata/metadatasvc.go rename metadatasvc/main.go => rkt/metadatasvc.go (61%) create mode 100644 stage1/init/registration.go diff --git a/build b/build index 7afbb57..039d540 100755 --- a/build +++ b/build @@ -47,7 +47,3 @@ fi echo "Building rkt (stage0)..." go build -o $GOBIN/rkt ${REPO_PATH}/rkt - -echo "Building metadatasvc..." -go build -o $GOBIN/metadatasvc ${REPO_PATH}/metadatasvc - diff --git a/metadata/metadatasvc.go b/metadata/metadatasvc.go new file mode 100644 index 0000000..3155acb --- /dev/null +++ b/metadata/metadatasvc.go @@ -0,0 +1,17 @@ +package metadata + +import "fmt" + +const ( + SvcIP = "169.254.169.255" + SvcPubPort = 80 + SvcPrvPort = 2375 +) + +func SvcPrvURL() string { + return fmt.Sprintf("http://127.0.0.1:%v", SvcPrvPort) +} + +func SvcPubURL() string { + return fmt.Sprintf("http://%v:%v", SvcIP, SvcPubPort) +} diff --git a/networking/net-plugin.go b/networking/net-plugin.go index 4931189..b81029f 100644 --- a/networking/net-plugin.go +++ b/networking/net-plugin.go @@ -79,21 +79,21 @@ func (e *containerEnv) execNetPlugin(cmd string, n *Net, netns, args, ifName str } vars := [][2]string{ - { "RKT_NETPLUGIN_COMMAND", cmd }, - { "RKT_NETPLUGIN_CONTID", e.contID.String() }, - { "RKT_NETPLUGIN_NETNS", netns }, - { "RKT_NETPLUGIN_ARGS", args }, - { "RKT_NETPLUGIN_IFNAME", ifName }, - { "RKT_NETPLUGIN_NETNAME", n.Name }, - { "RKT_NETPLUGIN_NETCONF", n.Filename }, + {"RKT_NETPLUGIN_COMMAND", cmd}, + {"RKT_NETPLUGIN_CONTID", e.contID.String()}, + {"RKT_NETPLUGIN_NETNS", netns}, + {"RKT_NETPLUGIN_ARGS", args}, + {"RKT_NETPLUGIN_IFNAME", ifName}, + {"RKT_NETPLUGIN_NETNAME", n.Name}, + {"RKT_NETPLUGIN_NETCONF", n.Filename}, } stdout := &bytes.Buffer{} c := exec.Cmd{ - Path: pluginPath, - Args: []string{pluginPath}, - Env: envVars(vars), + Path: pluginPath, + Args: []string{pluginPath}, + Env: envVars(vars), Stdout: stdout, Stderr: os.Stderr, } diff --git a/networking/net.go b/networking/net.go index 2835f83..bae6533 100644 --- a/networking/net.go +++ b/networking/net.go @@ -33,6 +33,7 @@ type Net struct { // Absolute path where users place their net configs const UserNetPath = "/etc/rkt/net.d" + // Default net path relative to stage1 root const DefaultNetPath = "etc/rkt/net.d/99-default.conf" diff --git a/networking/networking.go b/networking/networking.go index 1a95bfa..db86dd0 100644 --- a/networking/networking.go +++ b/networking/networking.go @@ -43,8 +43,8 @@ type activeNet struct { // describing the environment in which the container // is running in type containerEnv struct { - rktRoot string - contID types.UUID + rktRoot string + contID types.UUID } // Networking describes the networking details of a container. @@ -66,7 +66,7 @@ func Setup(rktRoot string, contID types.UUID) (*Networking, error) { n := Networking{ containerEnv: containerEnv{ rktRoot: rktRoot, - contID: contID, + contID: contID, }, } diff --git a/networking/plugins/veth/veth.go b/networking/plugins/veth/veth.go index 1ba68a0..7816b21 100644 --- a/networking/plugins/veth/veth.go +++ b/networking/plugins/veth/veth.go @@ -125,7 +125,7 @@ func main() { cmd := os.Getenv("RKT_NETPLUGIN_COMMAND") contID := os.Getenv("RKT_NETPLUGIN_CONTID") netns := os.Getenv("RKT_NETPLUGIN_NETNS") - args := os.Getenv("RKT_NETPLUGIN_ARGS") + args := os.Getenv("RKT_NETPLUGIN_ARGS") ifName := os.Getenv("RKT_NETPLUGIN_IFNAME") netConf := os.Getenv("RKT_NETPLUGIN_NETCONF") diff --git a/metadatasvc/main.go b/rkt/metadatasvc.go similarity index 61% rename from metadatasvc/main.go rename to rkt/metadatasvc.go index 996cdf6..e85c88d 100644 --- a/metadatasvc/main.go +++ b/rkt/metadatasvc.go @@ -23,55 +23,69 @@ import ( "fmt" "io" "log" + "net" "net/http" "net/url" + "os" "os/exec" + "strconv" "strings" "github.com/appc/spec/schema" "github.com/appc/spec/schema/types" "github.com/coreos/rocket/Godeps/_workspace/src/github.com/gorilla/mux" + + "github.com/coreos/rocket/metadata" ) -type metadata struct { +var ( + cmdMetadataSvc = &Command{ + Name: "metadatasvc", + Summary: "Run metadata service", + Usage: "[--src-addr CIDR] [--listen-port PORT] [--no-idle]", + Run: runMetadataSvc, + } +) + +type container struct { manifest schema.ContainerRuntimeManifest apps map[string]*schema.ImageManifest + ip string } var ( - metadataByIP = make(map[string]*metadata) - metadataByUID = make(map[types.UUID]*metadata) - hmacKey [sha256.Size]byte + containerByIP = make(map[string]*container) + containerByUID = make(map[types.UUID]*container) + hmacKey [sha256.Size]byte + + flagListenPort int + flagSrcAddrs string + flagNoIdle bool + + exitCh chan bool ) const ( - myPort = "4444" - metaIP = "169.254.169.255" - metaPort = "80" + listenFdsStart = 3 ) -func setupIPTables() error { +func init() { + commands = append(commands, cmdMetadataSvc) + cmdMetadataSvc.Flags.StringVar(&flagSrcAddrs, "src-addr", "0.0.0.0/0", "source address/range for iptables") + cmdMetadataSvc.Flags.IntVar(&flagListenPort, "listen-port", metadata.SvcPrvPort, "listen port") + cmdMetadataSvc.Flags.BoolVar(&flagNoIdle, "no-idle", false, "exit when last container is unregistered") +} + +func modifyIPTables(action, port string) error { return exec.Command( "iptables", "-t", "nat", - "-A", "PREROUTING", + action, "PREROUTING", "-p", "tcp", - "-d", metaIP, - "--dport", metaPort, + "-d", metadata.SvcIP, + "--dport", strconv.Itoa(metadata.SvcPubPort), "-j", "REDIRECT", - "--to-port", myPort, - ).Run() -} - -func antiSpoof(brPort, ipAddr string) error { - return exec.Command( - "ebtables", - "-t", "filter", - "-I", "INPUT", - "-i", brPort, - "-p", "IPV4", - "!", "--ip-source", ipAddr, - "-j", "DROP", + "--to-port", port, ).Run() } @@ -84,51 +98,70 @@ func queryValue(u *url.URL, key string) string { } func handleRegisterContainer(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + remoteIP := strings.Split(r.RemoteAddr, ":")[0] - if _, ok := metadataByIP[remoteIP]; ok { + if _, ok := containerByIP[remoteIP]; ok { // not allowed from container IP w.WriteHeader(http.StatusForbidden) return } - containerIP := queryValue(r.URL, "container_ip") + containerIP := queryValue(r.URL, "ip") if containerIP == "" { w.WriteHeader(http.StatusBadRequest) - fmt.Print(w, "container_ip missing") - return - } - containerBrPort := queryValue(r.URL, "container_brport") - if containerBrPort == "" { - w.WriteHeader(http.StatusBadRequest) - fmt.Print(w, "container_brport missing") + fmt.Fprint(w, "ip missing") return } - m := &metadata{ + c := &container{ apps: make(map[string]*schema.ImageManifest), + ip: containerIP, } - if err := json.NewDecoder(r.Body).Decode(&m.manifest); err != nil { + if err := json.NewDecoder(r.Body).Decode(&c.manifest); err != nil { w.WriteHeader(http.StatusBadRequest) fmt.Fprintf(w, "JSON-decoding failed: %v", err) return } - if err := antiSpoof(containerBrPort, containerIP); err != nil { - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "failed to set anti-spoofing: %v", err) - return - } - - metadataByIP[containerIP] = m - metadataByUID[m.manifest.UUID] = m + containerByIP[containerIP] = c + containerByUID[c.manifest.UUID] = c w.WriteHeader(http.StatusOK) } +func handleUnregisterContainer(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + uid, err := types.NewUUID(mux.Vars(r)["uid"]) + if err != nil { + w.WriteHeader(http.StatusNotFound) + fmt.Fprintf(w, "UUID is missing or malformed: %v", err) + return + } + + c, ok := containerByUID[*uid] + if !ok { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, "Container with given UUID not found") + return + } + + delete(containerByUID, *uid) + delete(containerByIP, c.ip) + w.WriteHeader(http.StatusOK) + + if flagNoIdle && len(containerByUID) == 0 { + exitCh <- true + } +} + func handleRegisterApp(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + remoteIP := strings.Split(r.RemoteAddr, ":")[0] - if _, ok := metadataByIP[remoteIP]; ok { + if _, ok := containerByIP[remoteIP]; ok { // not allowed from container IP w.WriteHeader(http.StatusForbidden) return @@ -141,7 +174,7 @@ func handleRegisterApp(w http.ResponseWriter, r *http.Request) { return } - m, ok := metadataByUID[*uid] + c, ok := containerByUID[*uid] if !ok { w.WriteHeader(http.StatusNotFound) fmt.Fprint(w, "Container with given UUID not found") @@ -157,31 +190,31 @@ func handleRegisterApp(w http.ResponseWriter, r *http.Request) { return } - m.apps[an] = app + c.apps[an] = app w.WriteHeader(http.StatusOK) } -func containerGet(h func(w http.ResponseWriter, r *http.Request, m *metadata)) http.HandlerFunc { +func containerGet(h func(w http.ResponseWriter, r *http.Request, c *container)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { remoteIP := strings.Split(r.RemoteAddr, ":")[0] - m, ok := metadataByIP[remoteIP] + c, ok := containerByIP[remoteIP] if !ok { w.WriteHeader(http.StatusNotFound) - fmt.Fprintf(w, "metadata by remoteIP (%v) not found", remoteIP) + fmt.Fprintf(w, "container by remoteIP (%v) not found", remoteIP) return } - h(w, r, m) + h(w, r, c) } } -func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, _ *schema.ImageManifest)) http.HandlerFunc { - return containerGet(func(w http.ResponseWriter, r *http.Request, m *metadata) { +func appGet(h func(w http.ResponseWriter, r *http.Request, c *container, _ *schema.ImageManifest)) http.HandlerFunc { + return containerGet(func(w http.ResponseWriter, r *http.Request, c *container) { appname := mux.Vars(r)["app"] - if im, ok := m.apps[appname]; ok { - h(w, r, m, im) + if im, ok := c.apps[appname]; ok { + h(w, r, c, im) } else { w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "App (%v) not found", appname) @@ -189,16 +222,20 @@ func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, _ *schem }) } -func handleContainerAnnotations(w http.ResponseWriter, r *http.Request, m *metadata) { +func handleContainerAnnotations(w http.ResponseWriter, r *http.Request, c *container) { + defer r.Body.Close() + w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) - for k := range m.manifest.Annotations { + for k := range c.manifest.Annotations { fmt.Fprintln(w, k) } } -func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metadata) { +func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, c *container) { + defer r.Body.Close() + k, err := types.NewACName(mux.Vars(r)["name"]) if err != nil { w.WriteHeader(http.StatusNotFound) @@ -206,7 +243,7 @@ func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metada return } - v, ok := m.manifest.Annotations.Get(k.String()) + v, ok := c.manifest.Annotations.Get(k.String()) if !ok { w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "Container annotation (%v) not found", k) @@ -218,17 +255,21 @@ func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metada w.Write([]byte(v)) } -func handleContainerManifest(w http.ResponseWriter, r *http.Request, m *metadata) { +func handleContainerManifest(w http.ResponseWriter, r *http.Request, c *container) { + defer r.Body.Close() + w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(m.manifest); err != nil { - fmt.Println(err) + if err := json.NewEncoder(w).Encode(c.manifest); err != nil { + log.Print(err) } } -func handleContainerUID(w http.ResponseWriter, r *http.Request, m *metadata) { - uid := m.manifest.UUID.String() +func handleContainerUID(w http.ResponseWriter, r *http.Request, c *container) { + defer r.Body.Close() + + uid := c.manifest.UUID.String() w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) @@ -251,16 +292,20 @@ func mergeAppAnnotations(im *schema.ImageManifest, cm *schema.ContainerRuntimeMa return merged } -func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) { +func handleAppAnnotations(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) { + defer r.Body.Close() + w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) - for _, annot := range mergeAppAnnotations(im, &m.manifest) { + for _, annot := range mergeAppAnnotations(im, &c.manifest) { fmt.Fprintln(w, string(annot.Name)) } } -func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) { +func handleAppAnnotation(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) { + defer r.Body.Close() + k, err := types.NewACName(mux.Vars(r)["name"]) if err != nil { w.WriteHeader(http.StatusNotFound) @@ -268,7 +313,7 @@ func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im return } - merged := mergeAppAnnotations(im, &m.manifest) + merged := mergeAppAnnotations(im, &c.manifest) v, ok := merged.Get(k.String()) if !ok { @@ -282,19 +327,23 @@ func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im w.Write([]byte(v)) } -func handleImageManifest(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) { +func handleImageManifest(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) { + defer r.Body.Close() + w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusOK) if err := json.NewEncoder(w).Encode(*im); err != nil { - fmt.Println(err) + log.Print(err) } } -func handleAppID(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) { +func handleAppID(w http.ResponseWriter, r *http.Request, c *container, im *schema.ImageManifest) { + defer r.Body.Close() + w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) - a := m.manifest.Apps.Get(im.Name) + a := c.manifest.Apps.Get(im.Name) if a == nil { panic("could not find app in manifest!") } @@ -317,8 +366,10 @@ func digest(r io.Reader) ([]byte, error) { } func handleContainerSign(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + remoteIP := strings.Split(r.RemoteAddr, ":")[0] - m, ok := metadataByIP[remoteIP] + c, ok := containerByIP[remoteIP] if !ok { w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "Metadata by remoteIP (%v) not found", remoteIP) @@ -335,7 +386,7 @@ func handleContainerSign(w http.ResponseWriter, r *http.Request) { // HMAC(UID:digest) h := hmac.New(sha256.New, hmacKey[:]) - h.Write(m.manifest.UUID[:]) + h.Write(c.manifest.UUID[:]) h.Write(d) // Send back digest:HMAC as the signature @@ -348,6 +399,8 @@ func handleContainerSign(w http.ResponseWriter, r *http.Request) { } func handleContainerVerify(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + uid, err := types.NewUUID(r.FormValue("uid")) if err != nil { w.WriteHeader(http.StatusBadRequest) @@ -398,21 +451,14 @@ func logReq(h func(w http.ResponseWriter, r *http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { resp := &httpResp{w, 0} h(resp, r) - fmt.Printf("%v %v - %v\n", r.Method, r.RequestURI, resp.status) + log.Printf("%v %v - %v", r.Method, r.RequestURI, resp.status) } } -func main() { - if err := setupIPTables(); err != nil { - log.Fatal(err) - } - - if err := initCrypto(); err != nil { - log.Fatal(err) - } - +func makeHandlers() http.Handler { r := mux.NewRouter() r.HandleFunc("/containers/", logReq(handleRegisterContainer)).Methods("POST") + r.HandleFunc("/containers/{uid}", logReq(handleUnregisterContainer)).Methods("DELETE") r.HandleFunc("/containers/{uid}/{app:.*}", logReq(handleRegisterApp)).Methods("PUT") acRtr := r.Headers("Metadata-Flavor", "AppContainer"). @@ -433,5 +479,83 @@ func main() { acRtr.HandleFunc("/container/hmac/sign", logReq(handleContainerSign)).Methods("POST") acRtr.HandleFunc("/container/hmac/verify", logReq(handleContainerVerify)).Methods("POST") - log.Fatal(http.ListenAndServe(":4444", r)) + return r +} + +func getListener() (net.Listener, error) { + s := os.Getenv("LISTEN_FDS") + if s != "" { + // socket activated + lfds, err := strconv.ParseInt(s, 10, 16) + if err != nil { + return nil, fmt.Errorf("Error parsing LISTEN_FDS env var: %v", err) + } + if lfds < 1 { + return nil, fmt.Errorf("LISTEN_FDS < 1") + } + + return net.FileListener(os.NewFile(uintptr(listenFdsStart), "listen")) + } else { + return net.Listen("tcp4", fmt.Sprintf(":%v", flagListenPort)) + } +} + +func cleanup(port string) { + if err := modifyIPTables("-D", port); err != nil { + log.Printf("Error cleaning up iptables: %v", err) + } +} + +func runMetadataSvc(args []string) (exit int) { + log.Print("Metadatasvc starting...") + + l, err := getListener() + if err != nil { + log.Printf("Error getting listener: %v", err) + return + } + + initCrypto() + + port := strings.Split(l.Addr().String(), ":")[1] + + if flagNoIdle { + // TODO(eyakubovich): this is very racy + // It's possible for last container to get unregistered + // and svc gets flagged to shutdown. Then another container + // starts to launch, sees that port is in use and doesn't + // start metadata svc only for this one to exit a moment later. + // However, --no-idle is meant for demos and having a single + // container spawn up (via --spawn-metadata-svc). The design + // of metadata svc is also likely to change as we convert it + // to be backed by persistent storage. + exitCh = make(chan bool, 1) + // wait for signal and exit + go func() { + <-exitCh + cleanup(port) + os.Exit(0) + }() + } + + if err := modifyIPTables("-A", port); err != nil { + log.Printf("Error setting up iptables: %v", err) + return 1 + } + + srv := http.Server{ + Handler: makeHandlers(), + } + + log.Print("Metadatasvc running...") + + if err = srv.Serve(l); err != nil { + log.Printf("Error serving HTTP: %v", err) + exit = 1 + } + + cleanup(port) + log.Print("Metadatasvc exiting...") + + return } diff --git a/rkt/rkt.go b/rkt/rkt.go index 20d1a91..9dd7305 100644 --- a/rkt/rkt.go +++ b/rkt/rkt.go @@ -17,6 +17,7 @@ package main import ( "flag" "fmt" + "log" "os" "path/filepath" "text/tabwriter" @@ -93,6 +94,11 @@ func main() { fmt.Fprintf(os.Stderr, "Run '%v help' for usage.\n", cliName) os.Exit(2) } + + if globalFlags.Debug { + log.SetOutput(os.Stderr) + } + os.Exit(cmd.Run(cmd.Flags.Args())) } diff --git a/rkt/run.go b/rkt/run.go index 549ffa5..83cf707 100644 --- a/rkt/run.go +++ b/rkt/run.go @@ -31,11 +31,12 @@ import ( ) var ( - flagStage1Init string - flagStage1Rootfs string - flagVolumes volumeMap - flagPrivateNet bool - cmdRun = &Command{ + flagStage1Init string + flagStage1Rootfs string + flagVolumes volumeMap + flagPrivateNet bool + flagSpawnMetadataSvc bool + cmdRun = &Command{ Name: "run", Summary: "Run image(s) in an application container in rocket", Usage: "[--volume LABEL:SOURCE] IMAGE...", @@ -51,6 +52,7 @@ func init() { cmdRun.Flags.StringVar(&flagStage1Rootfs, "stage1-rootfs", "", "path to stage1 rootfs tarball override") cmdRun.Flags.Var(&flagVolumes, "volume", "volumes to mount into the shared container environment") cmdRun.Flags.BoolVar(&flagPrivateNet, "private-net", false, "give container a private network") + cmdRun.Flags.BoolVar(&flagSpawnMetadataSvc, "spawn-metadata-svc", false, "launch metadata svc if not running") flagVolumes = volumeMap{} } @@ -131,14 +133,15 @@ func runRun(args []string) (exit int) { } cfg := stage0.Config{ - Store: ds, - ContainersDir: containersDir(), - Debug: globalFlags.Debug, - Stage1Init: flagStage1Init, - Stage1Rootfs: flagStage1Rootfs, - Images: imgs, - Volumes: flagVolumes, - PrivateNet: flagPrivateNet, + Store: ds, + ContainersDir: containersDir(), + Debug: globalFlags.Debug, + Stage1Init: flagStage1Init, + Stage1Rootfs: flagStage1Rootfs, + Images: imgs, + Volumes: flagVolumes, + PrivateNet: flagPrivateNet, + SpawnMetadataSvc: flagSpawnMetadataSvc, } cdir, err := stage0.Setup(cfg) if err != nil { diff --git a/stage0/run.go b/stage0/run.go index ca9aa64..b78b9fd 100644 --- a/stage0/run.go +++ b/stage0/run.go @@ -64,9 +64,10 @@ type Config struct { Stage1Rootfs string // compressed bundle containing a rootfs for stage1 Debug bool // TODO(jonboulle): These images are partially-populated hashes, this should be clarified. - Images []types.Hash // application images - Volumes map[string]string // map of volumes that rocket can provide to applications - PrivateNet bool // container should have its own network stack + Images []types.Hash // application images + Volumes map[string]string // map of volumes that rocket can provide to applications + PrivateNet bool // container should have its own network stack + SpawnMetadataSvc bool // launch metadata service } func init() { @@ -76,10 +77,6 @@ func init() { // Setup sets up a filesystem for a container based on the given config. // The directory containing the filesystem is returned, and any error encountered. func Setup(cfg Config) (string, error) { - if cfg.Debug { - log.SetOutput(os.Stderr) - } - cuuid, err := types.NewUUID(uuid.New()) if err != nil { return "", fmt.Errorf("error creating UID: %v", err) @@ -207,6 +204,17 @@ func Run(cfg Config, dir string) { if cfg.Debug { args = append(args, "--debug") } + if cfg.SpawnMetadataSvc { + rktExe, err := os.Readlink("/proc/self/exe") + if err != nil { + log.Fatalf("failed to readlink /proc/self/exe: %v", err) + } + dbgFlag := "" + if cfg.Debug { + dbgFlag = " --debug" + } + args = append(args, fmt.Sprintf("--metadata-svc=%s%s metadatasvc --no-idle", rktExe, dbgFlag)) + } if cfg.PrivateNet { args = append(args, "--private-net") } diff --git a/stage1/init/container.go b/stage1/init/container.go index 1dd918f..9a8fa07 100644 --- a/stage1/init/container.go +++ b/stage1/init/container.go @@ -34,9 +34,11 @@ import ( // Container encapsulates a ContainerRuntimeManifest and ImageManifests type Container struct { - Root string // root directory where the container will be located - Manifest *schema.ContainerRuntimeManifest - Apps map[string]*schema.ImageManifest + Root string // root directory where the container will be located + Manifest *schema.ContainerRuntimeManifest + Apps map[string]*schema.ImageManifest + MetadataSvcURL string + Networks []string } // LoadContainer loads a Container Runtime Manifest (as prepared by stage0) and @@ -147,6 +149,8 @@ func (c *Container) appToSystemd(am *schema.ImageManifest, id types.Hash) error env := app.Environment env["AC_APP_NAME"] = name + env["AC_METADATA_URL"] = c.MetadataSvcURL + for ek, ev := range env { ee := fmt.Sprintf(`"%s=%s"`, ek, ev) opts = append(opts, newUnitOption("Service", "Environment", ee)) @@ -267,10 +271,6 @@ func (c *Container) appToNspawnArgs(am *schema.ImageManifest, id types.Hash) ([] for _, i := range am.App.Isolators { switch i.Name { - case "private-network": - if i.Val == "true" { - args = append(args, "--private-network") - } case "capabilities/bounding-set": capList := strings.Join(strings.Split(i.Val, " "), ",") args = append(args, "--capability="+capList) diff --git a/stage1/init/init.go b/stage1/init/init.go index 5b2e948..6d3aed4 100644 --- a/stage1/init/init.go +++ b/stage1/init/init.go @@ -22,12 +22,16 @@ import ( "flag" "fmt" "io" + "log" + "net" "os" "os/exec" "path/filepath" "runtime" + "strings" "syscall" + "github.com/coreos/rocket/metadata" "github.com/coreos/rocket/networking" "github.com/coreos/rocket/path" ) @@ -76,13 +80,15 @@ func mirrorLocalZoneInfo(root string) { } var ( - debug bool - privNet bool + debug bool + metadataSvc string + privNet bool ) func init() { flag.BoolVar(&debug, "debug", false, "Run in debug mode") flag.BoolVar(&privNet, "private-net", false, "Setup private network (WIP!)") + flag.StringVar(&metadataSvc, "metadata-svc", "", "Launch specified metadata svc") // this ensures that main runs only on main thread (thread group leader). // since namespace ops (unshare, setns) are done for a single thread, we @@ -99,6 +105,7 @@ func stage1() int { } mirrorLocalZoneInfo(c.Root) + c.MetadataSvcURL = metadata.SvcPubURL() if err = c.ContainerToSystemd(); err != nil { fmt.Fprintf(os.Stderr, "Failed to configure systemd: %v\n", err) @@ -135,6 +142,13 @@ func stage1() int { env = append(env, "LD_PRELOAD="+filepath.Join(path.Stage1RootfsPath(c.Root), "fakesdboot.so")) env = append(env, "LD_LIBRARY_PATH="+filepath.Join(path.Stage1RootfsPath(c.Root), "usr/lib")) + if metadataSvc != "" { + if err = launchMetadataSvc(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to launch metadata svc: %v\n", err) + return 6 + } + } + if privNet { // careful not to make another local err variable. // cmd.Run sets the one from parent scope @@ -151,6 +165,12 @@ func stage1() int { return 6 } + if err = registerContainer(c, n.MetadataIP); err != nil { + fmt.Fprintf(os.Stderr, "Failed to register container: %v\n", err) + return 6 + } + defer unregisterContainer(c) + cmd := exec.Cmd{ Path: args[0], Args: args, @@ -172,6 +192,41 @@ func stage1() int { return 0 } +func launchMetadataSvc() error { + log.Print("Launching metadatasvc: ", metadataSvc) + + // use socket activation protocol to avoid race-condition of + // service becoming ready + // TODO(eyakubovich): remove hard-coded port + l, err := net.ListenTCP("tcp4", &net.TCPAddr{Port: metadata.SvcPrvPort}) + if err != nil { + if err.(*net.OpError).Err.(*os.SyscallError).Err == syscall.EADDRINUSE { + // assume metadatasvc is already running + return nil + } + return err + } + + defer l.Close() + + lf, err := l.File() + if err != nil { + return err + } + + // parse metadataSvc into exe and args + args := strings.Split(metadataSvc, " ") + cmd := exec.Cmd{ + Path: args[0], + Args: args, + Env: append(os.Environ(), "LISTEN_FDS=1"), + ExtraFiles: []*os.File{lf}, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + return cmd.Start() +} + func main() { flag.Parse() // move code into stage1() helper so defered fns get run diff --git a/stage1/init/registration.go b/stage1/init/registration.go new file mode 100644 index 0000000..0817e27 --- /dev/null +++ b/stage1/init/registration.go @@ -0,0 +1,74 @@ +package main + +import ( + "fmt" + "io" + "net" + "net/http" + "os" + "path" + + "github.com/coreos/rocket/metadata" + rktpath "github.com/coreos/rocket/path" +) + +func registerContainer(c *Container, ip net.IP) error { + cmf, err := os.Open(rktpath.ContainerManifestPath(c.Root)) + if err != nil { + return fmt.Errorf("failed opening runtime manifest: %v", err) + } + defer cmf.Close() + + pth := fmt.Sprintf("/containers/?ip=%v", ip.To4().String()) + if err := httpRequest("POST", pth, cmf); err != nil { + return fmt.Errorf("failed to register container with metadata svc: %v", err) + } + + uid := c.Manifest.UUID.String() + for _, app := range c.Manifest.Apps { + ampath := rktpath.ImageManifestPath(c.Root, app.ImageID) + amf, err := os.Open(ampath) + if err != nil { + fmt.Errorf("failed reading app manifest %q: %v", ampath, err) + } + defer amf.Close() + + if err := registerApp(uid, app.Name.String(), amf); err != nil { + fmt.Errorf("failed to register app with metadata svc: %v", err) + } + } + + return nil +} + +func unregisterContainer(c *Container) error { + pth := path.Join("/containers", c.Manifest.UUID.String()) + return httpRequest("DELETE", pth, nil) +} + +func registerApp(uuid, app string, r io.Reader) error { + pth := path.Join("/containers", uuid, app) + return httpRequest("PUT", pth, r) +} + +func httpRequest(method, pth string, body io.Reader) error { + uri := metadata.SvcPrvURL() + pth + req, err := http.NewRequest(method, uri, body) + if err != nil { + return err + } + + cli := http.Client{} + + resp, err := cli.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("%v %v returned %v", method, pth, resp.StatusCode) + } + + return nil +} diff --git a/test b/test index 7e4ffdb..f9d1ec1 100755 --- a/test +++ b/test @@ -15,7 +15,7 @@ COVER=${COVER:-"-cover"} source ./build TESTABLE_AND_FORMATTABLE="cas pkg/keystore pkg/lock pkg/tar rkt stage1/init" -FORMATTABLE="$TESTABLE_AND_FORMATTABLE metadatasvc path pkg/io pkg/proc stage0/run.go version" +FORMATTABLE="$TESTABLE_AND_FORMATTABLE networking path pkg/io pkg/proc stage0/run.go version" # user has not provided PKG override if [ -z "$PKG" ]; then