From cfb279da17188b6033aea78dc1b4570ebfb7d17d Mon Sep 17 00:00:00 2001 From: Matthew Johnson Date: Tue, 5 Dec 2017 15:49:42 -0800 Subject: [PATCH] Add Hashes and Pack structures Add hash calculation and internment functions. The point of using a map (hash table, associative array) to hold the SHA values is to improve speed. Having an O(n^2) constructor is fine for a test case of half a dozen unique values, but sucks if we have half a million. Wrote some trivial hash tests in a slightly unusual way to point out one of go's gotcha features, that range produces a copy of the value rather than a pointer to the value. Add hash and flag string getters for files. This implementation performs hash calculations on files only, not on links or directories. Signed-off-by: Icarus Sparry Signed-off-by: Matthew Johnson --- src/swupd/files.go | 61 +++++++++++++++++++++++- src/swupd/files_test.go | 61 ++++++++++++++++++++++++ src/swupd/hash.go | 28 +++++++++++ src/swupd/hash_test.go | 83 ++++++++++++++++++++++++++++++++ src/swupd/manifest.go | 12 +++-- src/swupd/packs.go | 10 ++++ src/swupd/testhash/dohash.go | 91 ++++++++++++++++++++++++++++++++++++ 7 files changed, 340 insertions(+), 6 deletions(-) create mode 100644 src/swupd/hash.go create mode 100644 src/swupd/hash_test.go create mode 100644 src/swupd/packs.go create mode 100644 src/swupd/testhash/dohash.go diff --git a/src/swupd/files.go b/src/swupd/files.go index 95cbb36..ba76b7b 100644 --- a/src/swupd/files.go +++ b/src/swupd/files.go @@ -1,6 +1,7 @@ package swupd import ( + "errors" "fmt" "os" ) @@ -18,6 +19,14 @@ const ( typeManifest ) +var typeBytes = map[ftype]byte{ + typeUnset: '.', + typeFile: 'F', + typeDirectory: 'D', + typeLink: 'L', + typeManifest: 'M', +} + const ( modifierUnset fmodifier = iota modifierConfig @@ -25,21 +34,39 @@ const ( modifierBoot ) +var modifierBytes = map[fmodifier]byte{ + modifierUnset: '.', + modifierConfig: 'C', + modifierState: 's', + modifierBoot: 'b', +} + const ( statusUnset fstatus = iota statusDeleted statusGhosted ) +var statusBytes = map[fstatus]byte{ + statusUnset: '.', + statusDeleted: 'd', + statusGhosted: 'g', +} + const ( renameUnset = false renameSet = true ) +var renameBytes = map[frename]byte{ + renameUnset: '.', + renameSet: 'r', +} + // File represents an entry in a manifest type File struct { Name string - Hash string + Hash hashval Version uint32 // flags @@ -142,3 +169,35 @@ func (f *File) setFlags(flags string) error { return nil } + +// setHash intern hashes of correct length and add index to f.Hash +func (f *File) setHash(hash string) error { + if len(hash) != 64 { + return fmt.Errorf("hash %v incorrect length", hash) + } + + f.Hash = internHash(hash) + return nil +} + +func (f *File) getHashString() string { + return *Hashes[f.Hash] +} + +func (f *File) getFlagString() (string, error) { + if f.Type == typeUnset && + f.Status == statusUnset && + f.Modifier == modifierUnset && + f.Rename == renameUnset { + return "", errors.New("no flags are set on file") + } + + flagBytes := []byte{ + typeBytes[f.Type], + statusBytes[f.Status], + modifierBytes[f.Modifier], + renameBytes[f.Rename], + } + + return string(flagBytes), nil +} diff --git a/src/swupd/files_test.go b/src/swupd/files_test.go index 39d3e66..9977352 100644 --- a/src/swupd/files_test.go +++ b/src/swupd/files_test.go @@ -196,3 +196,64 @@ func TestSetFlags(t *testing.T) { }) } } + +func TestSetHashValid(t *testing.T) { + // reset Hashes so we get the expected indices + Hashes = []*string{} + invHash = make(map[string]hashval) + f := File{} + validHash := "9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db7683cb574c36729" + if err := f.setHash(validHash); err != nil { + t.Error("setHash failed on valid hash") + } + + if f.Hash != 0 { + t.Errorf("f.Hash set to %v when 0 expected", f.Hash) + } +} + +func TestSetHashInvalid(t *testing.T) { + f := File{} + invalidHash := "9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db" + if err := f.setHash(invalidHash); err == nil { + t.Error("setHash did not fail on invalid hash") + } +} + +func TestGetHashString(t *testing.T) { + // reset Hashes so we get the expected indices + f := File{} + validHash := "9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db7683cb574c36729" + if err := f.setHash(validHash); err != nil { + t.Fatal("setHash failed on valid hash") + } + + hash := f.getHashString() + if hash != validHash { + t.Errorf("hash %v did not match expected %v", hash, validHash) + } +} + +func TestGetFlagString(t *testing.T) { + f := File{} + var err error + if err := f.setFlags("F.br"); err != nil { + t.Fatal(err) + } + + var flags string + if flags, err = f.getFlagString(); err != nil { + t.Error(err) + } + + if flags != "F.br" { + t.Errorf("%s did not match expected F.br", flags) + } +} + +func TestGetFlagStringFlagsUnset(t *testing.T) { + f := File{} + if _, err := f.getFlagString(); err == nil { + t.Error("getFlagString did not raise an error on unset flags") + } +} diff --git a/src/swupd/hash.go b/src/swupd/hash.go new file mode 100644 index 0000000..3677717 --- /dev/null +++ b/src/swupd/hash.go @@ -0,0 +1,28 @@ +package swupd + +type hashval int + +// Hashes is a global map of indices to hashes +var Hashes = []*string{} +var invHash = make(map[string]hashval) + +// internHash adds only new hashes to the Hashes slice and returns the index at +// which they are located +func internHash(hash string) hashval { + if key, ok := invHash[hash]; ok { + return key + } + Hashes = append(Hashes, &hash) + key := hashval(len(Hashes) - 1) + invHash[hash] = key + return key +} + +func (h hashval) String() string { + return *Hashes[int(h)] +} + +// HashEquals trivial equality function for hashval +func HashEquals(h1 hashval, h2 hashval) bool { + return h1 == h2 +} diff --git a/src/swupd/hash_test.go b/src/swupd/hash_test.go new file mode 100644 index 0000000..03a28be --- /dev/null +++ b/src/swupd/hash_test.go @@ -0,0 +1,83 @@ +package swupd + +import ( + "fmt" + "testing" +) + +func TestInternHash(t *testing.T) { + // reset Hashes so we get the expected indices + Hashes = []*string{} + testCases := []struct { + hash string + expected hashval + }{ + {"9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db7683cb574c36728", 0}, + {"33ccead640727d66c62be03e089a3ca3f4ef7c374a3eeab79764f9509075b0d8", 1}, + {"33ccead640727d66c62be03e089a3ca3f4ef7c374a3eeab79764f9509075b0d8", 1}, + {"b26f85ffaf3595ecd9a8b1e0c894f1b9e6e3ed0e8c3f28bcde3d66e63bfedd4d", 2}, + {"a49e68b3e2230855586e9ffd1b2962a2282411a488b80e3bd65851f068394c0a", 3}, + {"a49e68b3e2230855586e9ffd1b2962a2282411a488b80e3bd65851f068394c0a", 3}, + {"a49e68b3e2230855586e9ffd1b2962a2282411a488b80e3bd65851f068394c0a", 3}, + {"864f78102661c05b61cafcb59785349fd2fb7a956ec00a77198fe5bc2432de76", 4}, + } + + for _, tc := range testCases { + t.Run("validHash", func(t *testing.T) { + if idx := internHash(tc.hash); idx != tc.expected { + t.Errorf("interned hash index %v did not match expected %v", + idx, tc.expected) + } + }) + } +} + +func TestHashPrinting(t *testing.T) { + s := "0000000000000000000000000000000000000000000000000000000000000001" + v := internHash(s) + sout := fmt.Sprintf("%v", v) + if sout != s { + t.Errorf("in and out of hashtable do not match\n\t%v\n\t%v", sout, s) + } +} + +func TestHashPrinting2(t *testing.T) { + s := []byte("0000000000000000000000000000000000000000000000000000000000000001") + v := internHash(string(s)) + s[0] = '1' + sout := fmt.Sprintf("%v", v) + if sout == string(s) { + t.Errorf("in and out of hashtable do not match\n\t%v\n\t%v", sout, s) + } +} + +func TestHashEqual(t *testing.T) { + someHashes := []struct { + hash string + val hashval + }{ + {"3a60eb03c76ce17f1d08e0b5844c0455f6136c9b4bd4dd54c98cad2783354635", 0}, + {"b4b9333757d79e1e766dbb5db3160108e907e110bd19cba4d1d4230b299d0eb", 0}, + {"99aff80fc35d08b36c69ed0340ea80805f0c1b81ba7c734db6434b29a24c8391", 0}, + } + for i, tc := range someHashes { + // subtle point here, need to use the array index, rather than + // setting tc.val as tc is a copy of the entry, not a pointer to it + // See https://golang.org/ref/spec#RangeClause + someHashes[i].val = internHash(tc.hash) + } + // do n^2 compares + for i := range someHashes { + tc := &someHashes[i] + for j := range someHashes { + tc2 := &someHashes[j] + if HashEquals(tc.val, tc2.val) != (i == j) { + t.Errorf("HashEquals returns incorrect result %d %d %v %v", + i, j, tc.hash, tc2.hash) + } + } + } +} + +// Tip, to generate random hash values use this. +// hexdump -n32 -e '32 "%02x" "\n"' /dev/random diff --git a/src/swupd/manifest.go b/src/swupd/manifest.go index 1cb1eeb..074862f 100644 --- a/src/swupd/manifest.go +++ b/src/swupd/manifest.go @@ -96,10 +96,6 @@ func readManifestFileEntry(fields []string, m *Manifest) error { return fmt.Errorf("invalid number of flags: %v", fflags) } - if len(fhash) != 64 { - return fmt.Errorf("invalid hash length: %v", fhash) - } - var parsed uint64 var err error // fver must be a valid uint32 @@ -110,7 +106,13 @@ func readManifestFileEntry(fields []string, m *Manifest) error { // create a file record var file *File - file = &File{Name: fname, Hash: fhash, Version: ver} + file = &File{Name: fname, Version: ver} + + // set the file hash + if err = file.setHash(fhash); err != nil { + return fmt.Errorf("invalid hash: %v", err) + } + // Set the flags using fflags if err = file.setFlags(fflags); err != nil { return fmt.Errorf("invalid flags: %v", err) diff --git a/src/swupd/packs.go b/src/swupd/packs.go new file mode 100644 index 0000000..f406f31 --- /dev/null +++ b/src/swupd/packs.go @@ -0,0 +1,10 @@ +package swupd + +// Pack is an object containing delta files and full files for downloads +type Pack struct { + Bundle string + FromVersion uint32 + ToVersion uint32 + FullFileCount uint32 + Manifest *Manifest +} diff --git a/src/swupd/testhash/dohash.go b/src/swupd/testhash/dohash.go new file mode 100644 index 0000000..0d2cedf --- /dev/null +++ b/src/swupd/testhash/dohash.go @@ -0,0 +1,91 @@ +// Hack to generate swupd hashes, without xattrs. +// +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "io/ioutil" + "os" + + "golang.org/x/sys/unix" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "Usage: %s name1 name2 ...\n", os.Args[0]) + return + } + if len(os.Args) == 2 { + fmt.Println(Hashcalc(os.Args[1])) + } else { + for _, filename := range os.Args[1:] { + fmt.Printf("%s\t%s\n", filename, + Hashcalc(filename)) + } + } +} + +func Hashcalc(filename string) string { + key, err := hmac_compute_key(filename) + if err != nil { + fmt.Fprintf(os.Stderr, "Error stating file '%s' %v\n", filename, err) + return "" + } + // Only handle files for now.. + data, err := ioutil.ReadFile(filename) + if err != nil { + fmt.Fprintf(os.Stderr, "Read error for '%s' %v\n", filename, err) + return "" + } + result := hmac_sha256_for_data(key, data) + return string(result[:]) +} + +// hmac_sha256_for_data returns an ascii string of hex digits +func hmac_sha256_for_data(key []byte, data []byte) []byte { + var result [64]byte + + mac := hmac.New(sha256.New, key) + mac.Write(data) + hex.Encode(result[:], mac.Sum(nil)) + return result[:] +} + +// This is what I want to have for the key +// type updatestat struct { +// st_mode uint64 +// st_uid uint64 +// st_gid uint64 +// st_rdev uint64 +// st_size uint64 +// } + +// set fills in a buffer with an int in little endian order +func set(out []byte, in int64) { + for i := range out { + out[i] = byte(in & 0xff) + in >>= 8 + } +} + +// return what should be an ascii string as an array of byte +func hmac_compute_key(filename string) ([]byte, error) { + // Create the key + updatestat := [40]byte{} + var info unix.Stat_t + if err := unix.Stat(filename, &info); err != nil { + return nil, err + } + set(updatestat[24:32], 0) + set(updatestat[0:8], int64(info.Mode)) + set(updatestat[8:16], int64(info.Uid)) + set(updatestat[16:24], int64(info.Gid)) + // 24:32 is rdev, but this is always zero + set(updatestat[32:40], int64(info.Size)) + // fmt.Printf("key is %v\n", updatestat) + key := hmac_sha256_for_data(updatestat[:], nil) + return key, nil +}