Parallelize delta creation

Build all of the deltas first using the full manifest, and then add them
to the packs instead of finding and creating deltas synchronously on a
per-bundle-basis.

Signed-off-by: Tudor Marcu <tudor.marcu@intel.com>

builder/builder.go

@@ -1881,6 +1881,12 @@ func (b *Builder) BuildDeltaPacks(from, to uint32, printReport bool) error {
 
 	bundleDir := filepath.Join(b.Config.Builder.ServerStateDir, "image")
 	fmt.Printf("Using %d workers\n", b.NumDeltaWorkers)
+	// Create all deltas first
+	err = swupd.CreateAllDeltas(outputDir, int(fromManifest.Header.Version), int(toManifest.Header.Version), b.NumDeltaWorkers)
+	if err != nil {
+		return err
+	}
+	// Create packs filling in any missing deltas
 	return createDeltaPacks(fromManifest, toManifest, printReport, outputDir, bundleDir, b.NumDeltaWorkers)
 }
 
@@ -1923,6 +1929,15 @@ func (b *Builder) BuildDeltaPacksPreviousVersions(prev, to uint32, printReport b
 	fmt.Printf("Found %d previous versions\n", len(previousManifests))
 
 	bundleDir := filepath.Join(b.Config.Builder.ServerStateDir, "image")
+	// Create all deltas for all previous versions first based on full manifests
+	for _, fromManifest := range previousManifests {
+		fmt.Println()
+		err = swupd.CreateAllDeltas(outputDir, int(fromManifest.Header.Version), int(toManifest.Header.Version), b.NumDeltaWorkers)
+		if err != nil {
+			return err
+		}
+	}
+	// Create any missing delta files and pack all deltas up
 	for _, fromManifest := range previousManifests {
 		fmt.Println()
 		err = createDeltaPacks(fromManifest, toManifest, printReport, outputDir, bundleDir, b.NumDeltaWorkers)

swupd/packs.go

@@ -21,6 +21,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strconv"
 )
 
 const debugPacks = false
@@ -74,6 +75,40 @@ func (state PackState) String() string {
 	return "invalid"
 }
 
+// CreateAllDeltas builds all of the deltas using the full manifest from one
+// version to the next. This allows better concurrency and the pack creation
+// code can just worry about adding pre-existing files to packs.
+func CreateAllDeltas(outputDir string, fromVersion, toVersion, numWorkers int) error {
+	fromFile := filepath.Join(outputDir, strconv.Itoa(fromVersion), "Manifest.full")
+	toFile := filepath.Join(outputDir, strconv.Itoa(toVersion), "Manifest.full")
+
+	fromManifest, err := ParseManifestFile(fromFile)
+	if err != nil {
+		return err
+	}
+	toManifest, err := ParseManifestFile(toFile)
+	if err != nil {
+		return err
+	}
+
+	if fromVersion >= toVersion {
+		return fmt.Errorf("fromManifest version (%d) must be smaller than toManifest version (%d)", fromVersion, toVersion)
+	}
+
+	var c config
+	c, err = getConfig(filepath.Join(outputDir, ".."))
+	if err != nil {
+		return err
+	}
+
+	_, err = createDeltasFromManifests(&c, fromManifest, toManifest, numWorkers)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // WritePack writes the pack between two Manifests, or a zero pack if fromManifest is
 // nil. The toManifest should always be non nil. The outputDir is used to pick deltas and
 // fullfiles. If not empty, chrootDir is tried first as a fast alternative to