diff --git a/swupd/create_manifests.go b/swupd/create_manifests.go index c26c385..bca20d8 100644 --- a/swupd/create_manifests.go +++ b/swupd/create_manifests.go @@ -253,6 +253,11 @@ func CreateManifests(version uint32, minVersion uint32, format uint, statedir st return nil, err } + // Bootstrap delta directory, so we can assume every version will have one. + if err = os.MkdirAll(filepath.Join(verOutput, "delta"), 0755); err != nil { + return nil, err + } + newMoM := Manifest{ Name: "MoM", Header: ManifestHeader{ diff --git a/swupd/delta.go b/swupd/delta.go index aa241fc..1fa3cde 100644 --- a/swupd/delta.go +++ b/swupd/delta.go @@ -3,17 +3,40 @@ package swupd import ( "fmt" "os" + "os/exec" "path/filepath" "sync" + "syscall" "github.com/clearlinux/mixer-tools/helpers" "github.com/pkg/errors" ) -// CreateDeltas creates all delta files between the previous and current -// version of the supplied manifest, returning a list of files that -// delta creation failed for. -func CreateDeltas(manifest, statedir string, from, to uint32) ([]*File, error) { +const ( + // From swupd-server's include/swupd.h: + // + // Approximately the smallest size of a pair of input files which differ by a + // single bit that bsdiff can produce a more compact deltafile. Files smaller + // than this are always marked as different. See the magic 200 value in the + // bsdiff/src/diff.c code. + // + minimumSizeToMakeDeltaInBytes = 200 +) + +// Delta represents a delta file between two other files. If Error is present, it +// indicates that the delta couldn't be created. +type Delta struct { + Path string + Error error + from *File + to *File +} + +// CreateDeltas creates all delta files between the previous and current version of the +// supplied manifest. Returns a list of deltas (which contains information about +// individual delta errors). Returns error (and no deltas) if it can't assemble the delta +// list. +func CreateDeltas(manifest, statedir string, from, to uint32) ([]Delta, error) { var c config c, err := getConfig(statedir) @@ -31,149 +54,122 @@ func CreateDeltas(manifest, statedir string, from, to uint32) ([]*File, error) { return nil, err } - // Must be sorted before passing into linkPeersAndChange() - oldManifest.sortFilesName() - newManifest.sortFilesName() + return createDeltasFromManifests(&c, oldManifest, newManifest) +} - _, _, _ = newManifest.linkPeersAndChange(oldManifest, from) - - deltas, err := consolidateDeltaFiles(newManifest, from, c) +func createDeltasFromManifests(c *config, oldManifest, newManifest *Manifest) ([]Delta, error) { + deltas, err := findDeltas(c, oldManifest, newManifest) if err != nil { - return nil, errors.Wrapf(err, "Failed to create deltas list %s", manifest) + return nil, errors.Wrapf(err, "Failed to create deltas list %s", newManifest.Name) } + if len(deltas) == 0 { + return []Delta{}, nil + } + + // TODO: Unify these maxRoutines constants, get a better default (maybe NumCPUs) + // and make it configurable. const maxRoutines = 3 - var deltaQueue = make(chan *File) - var failedChan = make(chan *File) - var failedList = make(chan []*File) + var deltaQueue = make(chan *Delta) var wg sync.WaitGroup wg.Add(maxRoutines) - // Start a collector for the failure cases - go func() { - var failed []*File - // Drains the failedChan appropriately - for f := range failedChan { - failed = append(failed, f) - } - failedList <- failed // This list is what we return in the end - }() - - // Don't flood the system with goroutines, delta creation takes up an - // incredibly amount of memory, so we cannot max out on goroutines + // Delta creation takes a lot of memory, so create a limited amount of goroutines. for i := 0; i < maxRoutines; i++ { - go func(statedir string) { + go func() { defer wg.Done() - - // Take in jobs from the queue and try to make a delta - for f := range deltaQueue { - err := createDelta(c, f, statedir, f.DeltaPeer.Version, f.DeltaPeer.Hash) - if err != nil { - failedChan <- f - } + for delta := range deltaQueue { + delta.Error = createDelta(c, delta) } - }(statedir) + }() } - // Send jobs to the queue for delta goroutines to pick up - for _, file := range deltas { - deltaQueue <- file + // Send jobs to the queue for delta goroutines to pick up. + for i := range deltas { + deltaQueue <- &deltas[i] } // Send message that no more jobs are being sent close(deltaQueue) wg.Wait() - // Once wait finishes we can signal the collector that no more jobs exist - close(failedChan) - - return <-failedList, nil + return deltas, nil } -func consolidateDeltaFiles(manifest *Manifest, from uint32, c config) ([]*File, error) { - var files []*File +func createDelta(c *config, delta *Delta) error { + oldPath := filepath.Join(c.imageBase, fmt.Sprint(delta.from.Version), "full", delta.from.Name) + newPath := filepath.Join(c.imageBase, fmt.Sprint(delta.to.Version), "full", delta.to.Name) + dir := filepath.Join(c.outputDir, fmt.Sprint(delta.to.Version), "delta") + name := fmt.Sprintf("%d-%d-%s-%s", delta.from.Version, delta.to.Version, delta.from.Hash, delta.to.Hash) + delta.Path = filepath.Join(dir, name) - if manifest == nil { - return nil, nil + if _, err := os.Stat(delta.Path); err == nil { + // Skip existing deltas. Not verifying since client is resilient about that. + return nil } - for _, file := range manifest.Files { - if file.Version <= from { - continue - } - if file.DeltaPeer == nil { - continue - } - if file.Type != TypeFile || - file.DeltaPeer.Type != TypeFile { - continue - } - - deltadir := filepath.Join(c.outputDir, fmt.Sprintf("%d", file.Version), "delta") - fromToString := fmt.Sprintf("%d-%d-%s-%s", file.DeltaPeer.Version, file.Version, file.DeltaPeer.Hash, file.Hash) - - deltaFile := filepath.Join(deltadir, fromToString) - - // Only add to list if the delta file does not already exist on the system - if _, err := os.Stat(deltaFile); os.IsNotExist(err) { - // Only create deltas for files bigger than 200 bytes - fullname := filepath.Join(c.imageBase, fmt.Sprintf("%d", file.Version), "full", file.Name) - file.Info, _ = os.Stat(fullname) - if file.Info.Size() > 200 { - files = append(files, file) + if err := helpers.RunCommandSilent("bsdiff", oldPath, newPath, delta.Path); err != nil { + _ = os.Remove(delta.Path) + if exitErr, ok := errors.Cause(err).(*exec.ExitError); ok { + // bsdiff returns 1 that stands for "FULLDL", i.e. it decided that + // a delta is not worth. Give a better error message for that case. + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + if status == 1 { + return fmt.Errorf("bsdiff returned FULLDL, not using delta") + } } } - + return errors.Wrapf(err, "Failed to create delta for %s (%d) -> %s (%d)", delta.from.Name, delta.from.Version, delta.to.Name, delta.to.Version) } - return files, nil -} - -func createDelta(c config, file *File, statedir string, fromVersion uint32, fromHash Hashval) error { - // File without peers cannot have deltas - if file.DeltaPeer == nil { - return nil + // Check that the delta actually applies correctly. + testPath := delta.Path + ".testnewfile" + if err := helpers.RunCommandSilent("bspatch", oldPath, testPath, delta.Path); err != nil { + return errors.Wrapf(err, "Failed to apply delta %s", delta.Path) } + defer func() { + _ = os.Remove(testPath) + }() - // We only support deltas between two regular files for now - // TODO: Support directory -> file in the future, this is a complex case - if file.Type != TypeFile || file.DeltaPeer.Type != TypeFile { - return nil - } - - newfile := filepath.Join(c.imageBase, fmt.Sprintf("%d", file.Version), "full", file.Name) - original := filepath.Join(c.imageBase, fmt.Sprintf("%d", fromVersion), "full", file.Name) - - deltadir := filepath.Join(c.outputDir, fmt.Sprintf("%d", file.Version), "delta") - fromToString := fmt.Sprintf("%d-%d-%s-%s", fromVersion, file.Version, fromHash, file.Hash) - - // Files to create and validate deltas with - deltafile := filepath.Join(deltadir, fromToString) - testnewfile := filepath.Join(deltadir, "."+fromToString+".testnewfile") - - // Shell out to bsdiff for now...false = don't print to screen - if err := helpers.RunCommandSilent("bsdiff", original, newfile, deltafile); err != nil { - _ = os.Remove(deltafile) // Might have returned FULLDL - return errors.Wrapf(err, "Failed to create delta for %s -> %s", original, newfile) - } - - // Check that the delta actually applies correctly - if err := helpers.RunCommandSilent("bspatch", original, testnewfile, deltafile); err != nil { - return errors.Wrapf(err, "Failed to apply delta %s", deltafile) - } - - deltahash, err := Hashcalc(testnewfile) + testHash, err := Hashcalc(testPath) if err != nil { - return errors.Wrap(err, "Failed to calculate hash for new file") + _ = os.Remove(delta.Path) + return errors.Wrap(err, "Failed to calculate hash for test file created applying delta") } - - if !HashEquals(deltahash, file.Hash) { - return errors.Wrapf(err, "Delta mismatch: %s -> %s via delta: %s", original, newfile, deltafile) + if testHash != delta.to.Hash { + _ = os.Remove(delta.Path) + return errors.Wrapf(err, "Delta mismatch: %s -> %s via delta: %s", oldPath, newPath, delta.Path) } - _ = os.Remove(testnewfile) - - // add rename code here - return nil } + +func findDeltas(c *config, oldManifest, newManifest *Manifest) ([]Delta, error) { + oldManifest.sortFilesName() + newManifest.sortFilesName() + + err := linkDeltaPeers(c, oldManifest, newManifest) + if err != nil { + return nil, err + } + + deltaCount := 0 + for _, nf := range newManifest.Files { + if nf.DeltaPeer != nil { + deltaCount++ + } + } + + deltas := make([]Delta, 0, deltaCount) + for _, nf := range newManifest.Files { + if nf.DeltaPeer == nil { + continue + } + deltas = append(deltas, Delta{ + from: nf.DeltaPeer, + to: nf, + }) + } + + return deltas, nil +} diff --git a/swupd/delta_test.go b/swupd/delta_test.go index 902b4c3..e522844 100644 --- a/swupd/delta_test.go +++ b/swupd/delta_test.go @@ -21,6 +21,6 @@ func TestCreateDeltas(t *testing.T) { mustCreateManifestsStandard(t, 20, testDir) mustMkdir(t, filepath.Join(testDir, "www/20/delta")) - mustCreateDeltas(t, "Manifest.full", testDir, 10, 20) + mustCreateAllDeltas(t, "Manifest.full", testDir, 10, 20) mustExistDelta(t, testDir, "/bar", 10, 20) } diff --git a/swupd/helpers_test.go b/swupd/helpers_test.go index fcea32e..334315b 100644 --- a/swupd/helpers_test.go +++ b/swupd/helpers_test.go @@ -174,7 +174,7 @@ func resetHash() { } func mustMkdir(t *testing.T, name string) { - err := os.Mkdir(name, 0755) + err := os.MkdirAll(name, 0755) if err != nil { t.Fatal(err) } @@ -275,16 +275,20 @@ func checkManifestMatches(t *testing.T, testDir, ver, name string, res ...*regex } } -func mustCreateDeltas(t *testing.T, manifest, statedir string, from, to uint32) { - failed, err := CreateDeltas(manifest, statedir, from, to) +func mustCreateAllDeltas(t *testing.T, manifest, statedir string, from, to uint32) { + deltas, err := CreateDeltas(manifest, statedir, from, to) if err != nil { - if len(failed) > 0 { - t.Fatalf("%s: \n%d deltas did not get created for %s: %v", err, len(failed), manifest, failed) - return + t.Fatalf("couldn't create deltas for %s: %s", manifest, err) + } + + for _, d := range deltas { + if d.Error != nil { + t.Errorf("couldn't create delta for %s %d -> %s %d: %s", d.from.Name, d.from.Version, d.to.Name, d.to.Version, err) } - t.Fatal(err) - } else if len(failed) > 0 { - t.Fatalf("CreateDeltas succeeded but %d deltas did not get created for %s: %v", len(failed), manifest, failed) + } + + if t.Failed() { + t.Fatalf("couldn't create all deltas due to errors above") } } diff --git a/swupd/manifest.go b/swupd/manifest.go index c52f255..d23703a 100644 --- a/swupd/manifest.go +++ b/swupd/manifest.go @@ -16,7 +16,6 @@ package swupd import ( "bufio" - "errors" "fmt" "io" "os" @@ -27,6 +26,8 @@ import ( "strings" "text/template" "time" + + "github.com/pkg/errors" ) const manifestFieldDelim = "\t" @@ -474,6 +475,68 @@ func (m *Manifest) newDeleted(df *File) { m.Files = append(m.Files, df) } +// linkDeltaPeersForPack sets the DeltaPeer of the files in newManifest that have the corresponding files +// in oldManifest. +func linkDeltaPeersForPack(c *config, oldManifest, newManifest *Manifest) error { + newIndex := 0 + oldIndex := 0 + + for newIndex < len(newManifest.Files) && oldIndex < len(oldManifest.Files) { + nf := newManifest.Files[newIndex] + of := oldManifest.Files[oldIndex] + + switch { + case nf.Name < of.Name: + // Old file that is not in new manifest, no chance of delta. + newIndex++ + + case nf.Name > of.Name: + // New file that wasn't present before, no chance of delta. + oldIndex++ + + default: + newIndex++ + oldIndex++ + + // Matching names, we can have a delta if pass all the + // requirements below. + if nf.Version <= of.Version { + continue + } + + if nf.Hash == of.Hash { + continue + } + + if !nf.Present() || !nf.Present() { + continue + } + + if nf.Type != TypeFile || nf.Type != of.Type { + continue + } + + // Check file size to decide whether it should have a delta. + newPath := filepath.Join(c.imageBase, fmt.Sprint(nf.Version), "full", nf.Name) + fi, err := os.Stat(newPath) + if err != nil { + return errors.Wrapf(err, "error accessing %s to decide whether it can have a delta or not") + } + if fi.Size() < minimumSizeToMakeDeltaInBytes { + continue + } + + nf.DeltaPeer = of + of.DeltaPeer = nf + } + } + + // TODO: Once rename logic is in. Add code to identify what extra DeltaPeers + // should be created based on rename flag in newManifest. + + return nil +} + func (m *Manifest) readIncludes(bundles []*Manifest, c config) error { // read in //noship/-includes var err error diff --git a/swupd/packs.go b/swupd/packs.go index e7c7be0..9a6e2d7 100644 --- a/swupd/packs.go +++ b/swupd/packs.go @@ -60,6 +60,23 @@ type PackInfo struct { Warnings []string } +// Empty tells if the pack has contents or not. Packs without contents are not generated. +func (info PackInfo) Empty() bool { + return info.FullfileCount == 0 && info.DeltaCount == 0 +} + +func (state PackState) String() string { + switch state { + case NotPacked: + return "not packed" + case PackedDelta: + return "packed delta" + case PackedFullfile: + return "packed fullfile" + } + return "invalid" +} + // WritePack writes the pack between two Manifests, or a zero pack if fromManifest is // nil. The toManifest should always be non nil. The outputDir is used to pick deltas and // fullfiles. If not empty, chrootDir is tried first as a fast alternative to @@ -74,11 +91,35 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo toVersion := toManifest.Header.Version var fromVersion uint32 + var deltas []Delta if fromManifest != nil { fromVersion = fromManifest.Header.Version if fromVersion >= toVersion { return nil, fmt.Errorf("fromManifest version (%d) must be smaller than toManifest version (%d)", fromVersion, toVersion) } + + } + + if debugPacks { + log.Printf("DEBUG: WritePack for bundle %s from %d to %d", toManifest.Name, fromVersion, toVersion) + } + + if fromManifest != nil { + // TODO: Make WritePack itself take a Config. + var c config + c, err = getConfig(filepath.Join(outputDir, "..")) + if err != nil { + return nil, err + } + + deltas, err = createDeltasFromManifests(&c, fromManifest, toManifest) + if err != nil { + return nil, err + } + + if debugPacks { + log.Printf("DEBUG: %d potential deltas to use in pack", len(deltas)) + } } if debugPacks { @@ -89,7 +130,6 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo } } - // TODO: Should preallocate capacity. info = &PackInfo{ Entries: make([]PackEntry, len(toManifest.Files)), } @@ -107,6 +147,10 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo hashesInFrom = make(map[Hashval]bool) } + if debugPacks { + log.Printf("DEBUG: %d hashes already in from manifest", len(hashesInFrom)) + } + xw, err := NewExternalWriter(w, "xz") if err != nil { return nil, err @@ -142,6 +186,30 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo fullChrootDir = filepath.Join(chrootDir, fmt.Sprint(toVersion), "full") } + // Add all deltas that have not failed. + hasDelta := make(map[Hashval]bool) + for i := range deltas { + d := &deltas[i] + if d.Error != nil { + info.Warnings = append(info.Warnings, d.Error.Error()) + continue + } + var fallback bool + fallback, err = copyFromDelta(tw, d) + if err != nil { + // If copy from delta fails before writing to the pack, we can + // fallback to use the fullfile later. + if fallback { + info.Warnings = append(info.Warnings, err.Error()) + continue + } + return nil, err + } + + info.DeltaCount++ + hasDelta[d.to.Hash] = true + } + done := make(map[Hashval]bool) for i, f := range toManifest.Files { entry := &info.Entries[i] @@ -151,6 +219,10 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo entry.Reason = "already in from manifest" continue } + if hasDelta[f.Hash] { + entry.State = PackedDelta + continue + } if done[f.Hash] { entry.Reason = "hash already packed" continue @@ -166,7 +238,6 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo done[f.Hash] = true - // TODO: Pack deltas when available. entry.State = PackedFullfile entry.Reason = "from fullfile" info.FullfileCount++ @@ -174,6 +245,8 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo var fallback bool fallback, err = copyFromFullChrootFile(tw, fullChrootDir, f) if (err != nil) && fallback { + // If copy from chroot file fails before writing to the pack, we can + // fallback to try copying from the fullfile. info.Warnings = append(info.Warnings, err.Error()) err = copyFromFullfile(tw, outputDir, f) } else { @@ -187,6 +260,10 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo } } + if debugPacks { + log.Printf("DEBUG: pack created with %d fullfiles and %d deltas", info.FullfileCount, info.DeltaCount) + } + err = tw.Close() if err != nil { return nil, err @@ -194,6 +271,42 @@ func WritePack(w io.Writer, fromManifest, toManifest *Manifest, outputDir, chroo return info, nil } +func copyFromDelta(tw *tar.Writer, delta *Delta) (fallback bool, err error) { + f, err := os.Open(delta.Path) + if err != nil { + return true, err + } + defer func() { + _ = f.Close() + }() + fi, err := f.Stat() + if err != nil { + return true, err + } + if !fi.Mode().IsRegular() { + return true, fmt.Errorf("delta %s is not a regular file", delta.Path) + } + hdr, err := getHeaderFromFileInfo(fi) + if err != nil { + return true, err + } + hdr.Name = "delta/" + fi.Name() + hdr.Typeflag = tar.TypeReg + + // After we start writing on the tar writer, we can't let the caller fallback to another + // option anymore. + + err = tw.WriteHeader(hdr) + if err != nil { + return false, err + } + _, err = io.Copy(tw, f) + if err != nil { + return false, err + } + return false, nil +} + func copyFromFullChrootFile(tw *tar.Writer, fullChrootDir string, f *File) (fallback bool, err error) { realname := filepath.Join(fullChrootDir, f.Name) fi, err := os.Lstat(realname) @@ -428,6 +541,7 @@ func FindBundlesToPack(from *Manifest, to *Manifest) (map[string]*BundleToPack, // CreatePack creates the pack file for a specific bundle between two versions. The pack is written // in the TO version subdirectory of outputDir (e.g. a pack from 10 to 20 is written to "www/20"). +// Empty packs will lead to not creating the pack. func CreatePack(name string, fromVersion, toVersion uint32, outputDir, chrootDir string) (*PackInfo, error) { toDir := filepath.Join(outputDir, fmt.Sprint(toVersion)) toM, err := ParseManifestFile(filepath.Join(toDir, "Manifest."+name)) @@ -450,6 +564,7 @@ func CreatePack(name string, fromVersion, toVersion uint32, outputDir, chrootDir } info, err := WritePack(output, fromM, toM, outputDir, chrootDir) if err != nil { + _ = output.Close() _ = os.RemoveAll(packPath) return nil, err } @@ -459,5 +574,11 @@ func CreatePack(name string, fromVersion, toVersion uint32, outputDir, chrootDir return nil, err } + if info.Empty() { + // Don't bother leaving empty packs around. Not failing if remove fails since an + // empty pack is not incorrect. + _ = os.Remove(packPath) + } + return info, nil } diff --git a/swupd/packs_test.go b/swupd/packs_test.go index 3751f0e..9875b25 100644 --- a/swupd/packs_test.go +++ b/swupd/packs_test.go @@ -113,10 +113,13 @@ func TestFindBundlesToPack(t *testing.T) { m.Files = append(m.Files, bundle) } - sortAndPrintBundles := func(bundles []BundleToPack) { + sortBundles := func(bundles []BundleToPack) { sort.Slice(bundles, func(i, j int) bool { return bundles[i].Name < bundles[j].Name }) + } + + printBundles := func(bundles []BundleToPack) { for _, b := range bundles { fmt.Printf(" %s %d -> %d\n", b.Name, b.FromVersion, b.ToVersion) } @@ -158,14 +161,17 @@ func TestFindBundlesToPack(t *testing.T) { bundles = append(bundles, *b) } - fmt.Printf("== CASE: %s\n", tt.Name) - fmt.Printf("ACTUAL OUTPUT (%d bundles):\n", len(bundles)) - sortAndPrintBundles(bundles) - - fmt.Printf("EXPECTED OUTPUT (%d bundles):\n", len(tt.Expected)) - sortAndPrintBundles(tt.Expected) + sortBundles(bundles) + sortBundles(tt.Expected) if !reflect.DeepEqual(bundles, tt.Expected) { + fmt.Printf("== CASE: %s\n", tt.Name) + fmt.Printf("ACTUAL OUTPUT (%d bundles):\n", len(bundles)) + printBundles(bundles) + + fmt.Printf("EXPECTED OUTPUT (%d bundles):\n", len(tt.Expected)) + printBundles(tt.Expected) + t.Fatalf("mismatch between returned bundles to pack and expected bundles to pack in case %q", tt.Name) continue } @@ -248,6 +254,63 @@ func TestCreatePackZeroPacks(t *testing.T) { mustValidateZeroPack(t, ts.path("www/20/Manifest.shells"), ts.path("www/20/pack-shells-from-0.tar")) } +func TestCreatePackWithDelta(t *testing.T) { + fs := newTestFileSystem(t, "create-pack-") + defer fs.cleanup() + + const ( + format = 1 + minVer = 0 + ) + + // + // In version 10, create a bundle with files of different sizes. + // + emptyContents := "" + smallContents := "small" + largeContents := strings.Repeat("large", 1000) + if len(emptyContents) >= minimumSizeToMakeDeltaInBytes || len(smallContents) >= minimumSizeToMakeDeltaInBytes || len(largeContents) < minimumSizeToMakeDeltaInBytes { + t.Fatal("test contents sizes are invalid") + } + + mustInitStandardTest(t, fs.Dir, "0", "10", []string{"contents"}) + fs.write("image/10/contents/small1", emptyContents) + fs.write("image/10/contents/small2", smallContents) + fs.write("image/10/contents/large1", largeContents) + fs.write("image/10/contents/large2", largeContents) + mustCreateManifests(t, 10, minVer, format, fs.Dir) + + // + // In version 20, swap the content of small files, and modify the large files + // changing one byte or all bytes. + // + mustInitStandardTest(t, fs.Dir, "10", "20", []string{"contents"}) + fs.write("image/20/contents/small1", smallContents) + fs.write("image/20/contents/small2", smallContents) + fs.write("image/20/contents/large1", strings.ToUpper(largeContents[:1])+largeContents[1:]) + fs.write("image/20/contents/large2", largeContents[:1]+strings.ToUpper(largeContents[1:])) + mustCreateManifests(t, 20, minVer, format, fs.Dir) + + info := mustCreatePack(t, "contents", 10, 20, fs.path("www"), fs.path("image")) + mustHaveDeltaCount(t, info, 2) + + // + // In version 30, make a change to one large files from 20. + // + mustInitStandardTest(t, fs.Dir, "20", "30", []string{"contents"}) + fs.cp("image/20/contents", "image/30") + fs.write("image/30/contents/large1", strings.ToUpper(largeContents[:2])+largeContents[2:]) + mustCreateManifests(t, 30, minVer, format, fs.Dir) + + // Pack between 20 and 30 has only a delta for large1. + info = mustCreatePack(t, "contents", 20, 30, fs.path("www"), fs.path("image")) + mustHaveDeltaCount(t, info, 1) + + // Pack between 10 and 30 has both deltas. + info = mustCreatePack(t, "contents", 10, 30, fs.path("www"), fs.path("image")) + mustHaveDeltaCount(t, info, 2) +} + func TestCreatePackWithIncompleteChrootDir(t *testing.T) { fs := newTestFileSystem(t, "create-pack-") defer fs.cleanup() @@ -452,6 +515,7 @@ func mustCreatePack(t *testing.T, name string, fromVersion, toVersion uint32, ou func mustHaveFullfileCount(t *testing.T, info *PackInfo, expected uint64) { t.Helper() if info.FullfileCount != expected { + printPackInfo(info) t.Fatalf("pack has %d fullfiles but expected %d", info.FullfileCount, expected) } } @@ -459,6 +523,7 @@ func mustHaveFullfileCount(t *testing.T, info *PackInfo, expected uint64) { func mustHaveDeltaCount(t *testing.T, info *PackInfo, expected uint64) { t.Helper() if info.DeltaCount != expected { + printPackInfo(info) t.Fatalf("pack has %d deltas but expected %d", info.DeltaCount, expected) } } @@ -466,6 +531,7 @@ func mustHaveDeltaCount(t *testing.T, info *PackInfo, expected uint64) { func mustHaveNoWarnings(t *testing.T, info *PackInfo) { t.Helper() if len(info.Warnings) > 0 { + printPackInfo(info) t.Fatalf("unexpected warnings in pack: %s", strings.Join(info.Warnings, "\n")) } } @@ -477,3 +543,16 @@ func mustCreateFullfiles(t *testing.T, m *Manifest, chrootDir, outputDir string) t.Fatalf("couldn't create fullfiles: %s", err) } } + +func printPackInfo(info *PackInfo) { + fmt.Printf("WARNINGS (%d)\n", len(info.Warnings)) + for _, w := range info.Warnings { + fmt.Println(w) + } + fmt.Println() + fmt.Printf("ENTRIES (%d)\n", len(info.Entries)) + for _, e := range info.Entries { + fmt.Printf(" %-40s %s (%s)\n", e.File.Name, e.State, e.Reason) + } + fmt.Println() +}