Merge pull request #16147 from tiborvass/refactor-builder

Refactor builder with new Go interfaces

api/client/build.go

@@ -12,7 +12,6 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -131,13 +130,19 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 		return fmt.Errorf("cannot canonicalize dockerfile path %s: %v", relDockerfile, err)
 	}
 
-	var includes = []string{"."}
-
-	excludes, err := utils.ReadDockerIgnore(path.Join(contextDir, ".dockerignore"))
-	if err != nil {
+	f, err := os.Open(filepath.Join(contextDir, ".dockerignore"))
+	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
 
+	var excludes []string
+	if err == nil {
+		excludes, err = utils.ReadDockerIgnore(f)
+		if err != nil {
+			return err
+		}
+	}
+
 	if err := utils.ValidateContextDirectory(contextDir, excludes); err != nil {
 		return fmt.Errorf("Error checking context: '%s'.", err)
 	}
@@ -149,6 +154,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 	// removed.  The deamon will remove them for us, if needed, after it
 	// parses the Dockerfile. Ignore errors here, as they will have been
 	// caught by ValidateContextDirectory above.
+	var includes = []string{"."}
 	keepThem1, _ := fileutils.Matches(".dockerignore", excludes)
 	keepThem2, _ := fileutils.Matches(relDockerfile, excludes)
 	if keepThem1 || keepThem2 {

api/server/router/local/image.go

@@ -13,12 +13,17 @@ import (
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/builder/dockerfile"
 	"github.com/docker/docker/cliconfig"
+	"github.com/docker/docker/daemon/daemonbuilder"
 	"github.com/docker/docker/graph"
+	"github.com/docker/docker/graph/tags"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/parsers"
+	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/ulimit"
+	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/utils"
 	"golang.org/x/net/context"
@@ -46,7 +51,7 @@ func (s *router) postCommit(ctx context.Context, w http.ResponseWriter, r *http.
 		return err
 	}
 
-	commitCfg := &builder.CommitConfig{
+	commitCfg := &dockerfile.CommitConfig{
 		Pause:   pause,
 		Repo:    r.Form.Get("repo"),
 		Tag:     r.Form.Get("tag"),
@@ -56,13 +61,18 @@ func (s *router) postCommit(ctx context.Context, w http.ResponseWriter, r *http.
 		Config:  c,
 	}
 
-	imgID, err := builder.Commit(cname, s.daemon, commitCfg)
+	container, err := s.daemon.Get(cname)
+	if err != nil {
+		return err
+	}
+
+	imgID, err := dockerfile.Commit(container, s.daemon, commitCfg)
 	if err != nil {
 		return err
 	}
 
 	return httputils.WriteJSON(w, http.StatusCreated, &types.ContainerCommitResponse{
-		ID: imgID,
+		ID: string(imgID),
 	})
 }
 
@@ -125,7 +135,7 @@ func (s *router) postImagesCreate(ctx context.Context, w http.ResponseWriter, r
 		// generated from the download to be available to the output
 		// stream processing below
 		var newConfig *runconfig.Config
-		newConfig, err = builder.BuildFromConfig(s.daemon, &runconfig.Config{}, r.Form["changes"])
+		newConfig, err = dockerfile.BuildFromConfig(&runconfig.Config{}, r.Form["changes"])
 		if err != nil {
 			return err
 		}
@@ -269,7 +279,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 	var (
 		authConfigs        = map[string]cliconfig.AuthConfig{}
 		authConfigsEncoded = r.Header.Get("X-Registry-Config")
-		buildConfig        = builder.NewBuildConfig()
+		buildConfig        = &dockerfile.Config{}
 	)
 
 	if authConfigsEncoded != "" {
@@ -284,6 +294,21 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 	w.Header().Set("Content-Type", "application/json")
 
 	version := httputils.VersionFromContext(ctx)
+	output := ioutils.NewWriteFlusher(w)
+	sf := streamformatter.NewJSONStreamFormatter()
+	errf := func(err error) error {
+		// Do not write the error in the http output if it's still empty.
+		// This prevents from writing a 200(OK) when there is an interal error.
+		if !output.Flushed() {
+			return err
+		}
+		_, err = w.Write(sf.FormatError(errors.New(utils.GetErrorMessage(err))))
+		if err != nil {
+			logrus.Warnf("could not write error response: %v", err)
+		}
+		return nil
+	}
+
 	if httputils.BoolValue(r, "forcerm") && version.GreaterThanOrEqualTo("1.12") {
 		buildConfig.Remove = true
 	} else if r.FormValue("rm") == "" && version.GreaterThanOrEqualTo("1.12") {
@@ -295,17 +320,22 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 		buildConfig.Pull = true
 	}
 
-	output := ioutils.NewWriteFlusher(w)
-	buildConfig.Stdout = output
-	buildConfig.Context = r.Body
+	repoName, tag := parsers.ParseRepositoryTag(r.FormValue("t"))
+	if repoName != "" {
+		if err := registry.ValidateRepositoryName(repoName); err != nil {
+			return errf(err)
+		}
+		if len(tag) > 0 {
+			if err := tags.ValidateTagName(tag); err != nil {
+				return errf(err)
+			}
+		}
+	}
 
-	buildConfig.RemoteURL = r.FormValue("remote")
 	buildConfig.DockerfileName = r.FormValue("dockerfile")
-	buildConfig.RepoName = r.FormValue("t")
-	buildConfig.SuppressOutput = httputils.BoolValue(r, "q")
-	buildConfig.NoCache = httputils.BoolValue(r, "nocache")
+	buildConfig.Verbose = !httputils.BoolValue(r, "q")
+	buildConfig.UseCache = !httputils.BoolValue(r, "nocache")
 	buildConfig.ForceRemove = httputils.BoolValue(r, "forcerm")
-	buildConfig.AuthConfigs = authConfigs
 	buildConfig.MemorySwap = httputils.Int64ValueOrZero(r, "memswap")
 	buildConfig.Memory = httputils.Int64ValueOrZero(r, "memory")
 	buildConfig.CPUShares = httputils.Int64ValueOrZero(r, "cpushares")
@@ -319,7 +349,7 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 	ulimitsJSON := r.FormValue("ulimits")
 	if ulimitsJSON != "" {
 		if err := json.NewDecoder(strings.NewReader(ulimitsJSON)).Decode(&buildUlimits); err != nil {
-			return err
+			return errf(err)
 		}
 		buildConfig.Ulimits = buildUlimits
 	}
@@ -328,12 +358,50 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 	buildArgsJSON := r.FormValue("buildargs")
 	if buildArgsJSON != "" {
 		if err := json.NewDecoder(strings.NewReader(buildArgsJSON)).Decode(&buildArgs); err != nil {
-			return err
+			return errf(err)
 		}
+		buildConfig.BuildArgs = buildArgs
 	}
-	buildConfig.BuildArgs = buildArgs
 
-	// Job cancellation. Note: not all job types support this.
+	remoteURL := r.FormValue("remote")
+
+	// Currently, only used if context is from a remote url.
+	// The field `In` is set by DetectContextFromRemoteURL.
+	// Look at code in DetectContextFromRemoteURL for more information.
+	pReader := &progressreader.Config{
+		// TODO: make progressreader streamformatter-agnostic
+		Out:       output,
+		Formatter: sf,
+		Size:      r.ContentLength,
+		NewLines:  true,
+		ID:        "Downloading context",
+		Action:    remoteURL,
+	}
+
+	var (
+		context        builder.ModifiableContext
+		dockerfileName string
+		err            error
+	)
+	context, dockerfileName, err = daemonbuilder.DetectContextFromRemoteURL(r.Body, remoteURL, pReader)
+	if err != nil {
+		return errf(err)
+	}
+	defer func() {
+		if err := context.Close(); err != nil {
+			logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
+		}
+	}()
+
+	docker := daemonbuilder.Docker{s.daemon, output, authConfigs}
+
+	b, err := dockerfile.NewBuilder(buildConfig, docker, builder.DockerIgnoreContext{context}, nil)
+	if err != nil {
+		return errf(err)
+	}
+	b.Stdout = &streamformatter.StdoutFormatter{Writer: output, StreamFormatter: sf}
+	b.Stderr = &streamformatter.StderrFormatter{Writer: output, StreamFormatter: sf}
+
 	if closeNotifier, ok := w.(http.CloseNotifier); ok {
 		finished := make(chan struct{})
 		defer close(finished)
@@ -342,20 +410,26 @@ func (s *router) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R
 			case <-finished:
 			case <-closeNotifier.CloseNotify():
 				logrus.Infof("Client disconnected, cancelling job: build")
-				buildConfig.Cancel()
+				b.Cancel()
 			}
 		}()
 	}
 
-	if err := builder.Build(s.daemon, buildConfig); err != nil {
-		// Do not write the error in the http output if it's still empty.
-		// This prevents from writing a 200(OK) when there is an interal error.
-		if !output.Flushed() {
-			return err
-		}
-		sf := streamformatter.NewJSONStreamFormatter()
-		w.Write(sf.FormatError(errors.New(utils.GetErrorMessage(err))))
+	if len(dockerfileName) > 0 {
+		b.DockerfileName = dockerfileName
 	}
+
+	imgID, err := b.Build()
+	if err != nil {
+		return errf(err)
+	}
+
+	if repoName != "" {
+		if err := s.daemon.Repositories().Tag(repoName, tag, string(imgID), true); err != nil {
+			return errf(err)
+		}
+	}
+
 	return nil
 }