Add Hashes and Pack structures

Add hash calculation and internment functions.

The point of using a map (hash table, associative array) to hold the
SHA values is to improve speed. Having an O(n^2) constructor is fine
for a test case of half a dozen unique values, but sucks if we have
half a million.

Wrote some trivial hash tests in a slightly unusual way to point out one
of go's gotcha features, that range produces a copy of the value rather
than a pointer to the value.

Add hash and flag string getters for files.

This implementation performs hash calculations on files only, not on
links or directories.

Signed-off-by: Icarus Sparry <icarus.w.sparry@intel.com>
Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

src/swupd/files.go

@@ -1,6 +1,7 @@
 package swupd
 
 import (
+	"errors"
 	"fmt"
 	"os"
 )
@@ -18,6 +19,14 @@ const (
 	typeManifest
 )
 
+var typeBytes = map[ftype]byte{
+	typeUnset:     '.',
+	typeFile:      'F',
+	typeDirectory: 'D',
+	typeLink:      'L',
+	typeManifest:  'M',
+}
+
 const (
 	modifierUnset fmodifier = iota
 	modifierConfig
@@ -25,21 +34,39 @@ const (
 	modifierBoot
 )
 
+var modifierBytes = map[fmodifier]byte{
+	modifierUnset:  '.',
+	modifierConfig: 'C',
+	modifierState:  's',
+	modifierBoot:   'b',
+}
+
 const (
 	statusUnset fstatus = iota
 	statusDeleted
 	statusGhosted
 )
 
+var statusBytes = map[fstatus]byte{
+	statusUnset:   '.',
+	statusDeleted: 'd',
+	statusGhosted: 'g',
+}
+
 const (
 	renameUnset = false
 	renameSet   = true
 )
 
+var renameBytes = map[frename]byte{
+	renameUnset: '.',
+	renameSet:   'r',
+}
+
 // File represents an entry in a manifest
 type File struct {
 	Name    string
-	Hash    string
+	Hash    hashval
 	Version uint32
 
 	// flags
@@ -142,3 +169,35 @@ func (f *File) setFlags(flags string) error {
 
 	return nil
 }
+
+// setHash intern hashes of correct length and add index to f.Hash
+func (f *File) setHash(hash string) error {
+	if len(hash) != 64 {
+		return fmt.Errorf("hash %v incorrect length", hash)
+	}
+
+	f.Hash = internHash(hash)
+	return nil
+}
+
+func (f *File) getHashString() string {
+	return *Hashes[f.Hash]
+}
+
+func (f *File) getFlagString() (string, error) {
+	if f.Type == typeUnset &&
+		f.Status == statusUnset &&
+		f.Modifier == modifierUnset &&
+		f.Rename == renameUnset {
+		return "", errors.New("no flags are set on file")
+	}
+
+	flagBytes := []byte{
+		typeBytes[f.Type],
+		statusBytes[f.Status],
+		modifierBytes[f.Modifier],
+		renameBytes[f.Rename],
+	}
+
+	return string(flagBytes), nil
+}

src/swupd/files_test.go

@@ -196,3 +196,64 @@ func TestSetFlags(t *testing.T) {
 		})
 	}
 }
+
+func TestSetHashValid(t *testing.T) {
+	// reset Hashes so we get the expected indices
+	Hashes = []*string{}
+	invHash = make(map[string]hashval)
+	f := File{}
+	validHash := "9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db7683cb574c36729"
+	if err := f.setHash(validHash); err != nil {
+		t.Error("setHash failed on valid hash")
+	}
+
+	if f.Hash != 0 {
+		t.Errorf("f.Hash set to %v when 0 expected", f.Hash)
+	}
+}
+
+func TestSetHashInvalid(t *testing.T) {
+	f := File{}
+	invalidHash := "9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db"
+	if err := f.setHash(invalidHash); err == nil {
+		t.Error("setHash did not fail on invalid hash")
+	}
+}
+
+func TestGetHashString(t *testing.T) {
+	// reset Hashes so we get the expected indices
+	f := File{}
+	validHash := "9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db7683cb574c36729"
+	if err := f.setHash(validHash); err != nil {
+		t.Fatal("setHash failed on valid hash")
+	}
+
+	hash := f.getHashString()
+	if hash != validHash {
+		t.Errorf("hash %v did not match expected %v", hash, validHash)
+	}
+}
+
+func TestGetFlagString(t *testing.T) {
+	f := File{}
+	var err error
+	if err := f.setFlags("F.br"); err != nil {
+		t.Fatal(err)
+	}
+
+	var flags string
+	if flags, err = f.getFlagString(); err != nil {
+		t.Error(err)
+	}
+
+	if flags != "F.br" {
+		t.Errorf("%s did not match expected F.br", flags)
+	}
+}
+
+func TestGetFlagStringFlagsUnset(t *testing.T) {
+	f := File{}
+	if _, err := f.getFlagString(); err == nil {
+		t.Error("getFlagString did not raise an error on unset flags")
+	}
+}

src/swupd/hash.go

@@ -0,0 +1,28 @@
+package swupd
+
+type hashval int
+
+// Hashes is a global map of indices to hashes
+var Hashes = []*string{}
+var invHash = make(map[string]hashval)
+
+// internHash adds only new hashes to the Hashes slice and returns the index at
+// which they are located
+func internHash(hash string) hashval {
+	if key, ok := invHash[hash]; ok {
+		return key
+	}
+	Hashes = append(Hashes, &hash)
+	key := hashval(len(Hashes) - 1)
+	invHash[hash] = key
+	return key
+}
+
+func (h hashval) String() string {
+	return *Hashes[int(h)]
+}
+
+// HashEquals trivial equality function for hashval
+func HashEquals(h1 hashval, h2 hashval) bool {
+	return h1 == h2
+}

src/swupd/hash_test.go

@@ -0,0 +1,83 @@
+package swupd
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestInternHash(t *testing.T) {
+	// reset Hashes so we get the expected indices
+	Hashes = []*string{}
+	testCases := []struct {
+		hash     string
+		expected hashval
+	}{
+		{"9bcc1718757db298fb656ae6e2ee143dde746f49fbf6805db7683cb574c36728", 0},
+		{"33ccead640727d66c62be03e089a3ca3f4ef7c374a3eeab79764f9509075b0d8", 1},
+		{"33ccead640727d66c62be03e089a3ca3f4ef7c374a3eeab79764f9509075b0d8", 1},
+		{"b26f85ffaf3595ecd9a8b1e0c894f1b9e6e3ed0e8c3f28bcde3d66e63bfedd4d", 2},
+		{"a49e68b3e2230855586e9ffd1b2962a2282411a488b80e3bd65851f068394c0a", 3},
+		{"a49e68b3e2230855586e9ffd1b2962a2282411a488b80e3bd65851f068394c0a", 3},
+		{"a49e68b3e2230855586e9ffd1b2962a2282411a488b80e3bd65851f068394c0a", 3},
+		{"864f78102661c05b61cafcb59785349fd2fb7a956ec00a77198fe5bc2432de76", 4},
+	}
+
+	for _, tc := range testCases {
+		t.Run("validHash", func(t *testing.T) {
+			if idx := internHash(tc.hash); idx != tc.expected {
+				t.Errorf("interned hash index %v did not match expected %v",
+					idx, tc.expected)
+			}
+		})
+	}
+}
+
+func TestHashPrinting(t *testing.T) {
+	s := "0000000000000000000000000000000000000000000000000000000000000001"
+	v := internHash(s)
+	sout := fmt.Sprintf("%v", v)
+	if sout != s {
+		t.Errorf("in and out of hashtable do not match\n\t%v\n\t%v", sout, s)
+	}
+}
+
+func TestHashPrinting2(t *testing.T) {
+	s := []byte("0000000000000000000000000000000000000000000000000000000000000001")
+	v := internHash(string(s))
+	s[0] = '1'
+	sout := fmt.Sprintf("%v", v)
+	if sout == string(s) {
+		t.Errorf("in and out of hashtable do not match\n\t%v\n\t%v", sout, s)
+	}
+}
+
+func TestHashEqual(t *testing.T) {
+	someHashes := []struct {
+		hash string
+		val  hashval
+	}{
+		{"3a60eb03c76ce17f1d08e0b5844c0455f6136c9b4bd4dd54c98cad2783354635", 0},
+		{"b4b9333757d79e1e766dbb5db3160108e907e110bd19cba4d1d4230b299d0eb", 0},
+		{"99aff80fc35d08b36c69ed0340ea80805f0c1b81ba7c734db6434b29a24c8391", 0},
+	}
+	for i, tc := range someHashes {
+		// subtle point here, need to use the array index, rather than
+		// setting tc.val as tc is a copy of the entry, not a pointer to it
+		// See https://golang.org/ref/spec#RangeClause
+		someHashes[i].val = internHash(tc.hash)
+	}
+	// do n^2 compares
+	for i := range someHashes {
+		tc := &someHashes[i]
+		for j := range someHashes {
+			tc2 := &someHashes[j]
+			if HashEquals(tc.val, tc2.val) != (i == j) {
+				t.Errorf("HashEquals returns incorrect result %d %d %v %v",
+					i, j, tc.hash, tc2.hash)
+			}
+		}
+	}
+}
+
+// Tip, to generate random hash values use this.
+// hexdump -n32 -e '32 "%02x" "\n"' /dev/random

src/swupd/manifest.go

@@ -96,10 +96,6 @@ func readManifestFileEntry(fields []string, m *Manifest) error {
 		return fmt.Errorf("invalid number of flags: %v", fflags)
 	}
 
-	if len(fhash) != 64 {
-		return fmt.Errorf("invalid hash length: %v", fhash)
-	}
-
 	var parsed uint64
 	var err error
 	// fver must be a valid uint32
@@ -110,7 +106,13 @@ func readManifestFileEntry(fields []string, m *Manifest) error {
 
 	// create a file record
 	var file *File
-	file = &File{Name: fname, Hash: fhash, Version: ver}
+	file = &File{Name: fname, Version: ver}
+
+	// set the file hash
+	if err = file.setHash(fhash); err != nil {
+		return fmt.Errorf("invalid hash: %v", err)
+	}
+
 	// Set the flags using fflags
 	if err = file.setFlags(fflags); err != nil {
 		return fmt.Errorf("invalid flags: %v", err)

src/swupd/packs.go

@@ -0,0 +1,10 @@
+package swupd
+
+// Pack is an object containing delta files and full files for downloads
+type Pack struct {
+	Bundle        string
+	FromVersion   uint32
+	ToVersion     uint32
+	FullFileCount uint32
+	Manifest      *Manifest
+}

src/swupd/testhash/dohash.go

@@ -0,0 +1,91 @@
+// Hack to generate swupd hashes, without xattrs.
+//
+package main
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s name1 name2 ...\n", os.Args[0])
+		return
+	}
+	if len(os.Args) == 2 {
+		fmt.Println(Hashcalc(os.Args[1]))
+	} else {
+		for _, filename := range os.Args[1:] {
+			fmt.Printf("%s\t%s\n", filename,
+				Hashcalc(filename))
+		}
+	}
+}
+
+func Hashcalc(filename string) string {
+	key, err := hmac_compute_key(filename)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error stating file '%s' %v\n", filename, err)
+		return ""
+	}
+	// Only handle files for now..
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Read error for '%s' %v\n", filename, err)
+		return ""
+	}
+	result := hmac_sha256_for_data(key, data)
+	return string(result[:])
+}
+
+// hmac_sha256_for_data returns an ascii string of hex digits
+func hmac_sha256_for_data(key []byte, data []byte) []byte {
+	var result [64]byte
+
+	mac := hmac.New(sha256.New, key)
+	mac.Write(data)
+	hex.Encode(result[:], mac.Sum(nil))
+	return result[:]
+}
+
+// This is what I want to have for the key
+// type updatestat struct {
+// 	st_mode uint64
+// 	st_uid  uint64
+// 	st_gid  uint64
+// 	st_rdev uint64
+// 	st_size uint64
+// }
+
+// set fills in a buffer with an int in little endian order
+func set(out []byte, in int64) {
+	for i := range out {
+		out[i] = byte(in & 0xff)
+		in >>= 8
+	}
+}
+
+// return what should be an ascii string as an array of byte
+func hmac_compute_key(filename string) ([]byte, error) {
+	// Create the key
+	updatestat := [40]byte{}
+	var info unix.Stat_t
+	if err := unix.Stat(filename, &info); err != nil {
+		return nil, err
+	}
+	set(updatestat[24:32], 0)
+	set(updatestat[0:8], int64(info.Mode))
+	set(updatestat[8:16], int64(info.Uid))
+	set(updatestat[16:24], int64(info.Gid))
+	// 24:32 is rdev, but this is always zero
+	set(updatestat[32:40], int64(info.Size))
+	// fmt.Printf("key is %v\n", updatestat)
+	key := hmac_sha256_for_data(updatestat[:], nil)
+	return key, nil
+}