From b671aba659d2209b46484e14749108779f7198fd Mon Sep 17 00:00:00 2001 From: Tudor Marcu Date: Wed, 1 Mar 2017 01:34:05 -0800 Subject: [PATCH] Introduce new mixer tool The tools have been converted from many bash scripts into a single binary, written in Go. This creates a codebase that is much easier to edit and maintain, and a single tool that provides all the functionality needed. The interface is more intuitive and easy to use as it provides options and menus for each subcommand. Signed-off-by: Tudor Marcu --- .gitignore | 1 - Makefile | 33 +++ Makefile.gobuild | 42 ++++ src/builder/builder.go | 504 +++++++++++++++++++++++++++++++++++++++++ src/helpers/helpers.go | 232 +++++++++++++++++++ src/mixer/main.go | 151 ++++++++++++ 6 files changed, 962 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 Makefile.gobuild create mode 100644 src/builder/builder.go create mode 100644 src/helpers/helpers.go create mode 100644 src/mixer/main.go diff --git a/.gitignore b/.gitignore index 8ec2ed3..1c471a1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ build/ bundles repos/ results/ -Makefile Makefile.in configure aclocal.m4 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4e23d3b --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +PROJECT_ROOT := src/ +VERSION = 0.1 + +.DEFAULT_GOAL := all + +# Locate testables: +_TESTABLES = $(shell find src/ -name '*_test.go'|xargs -I{} dirname {}|sed 's/src\///g'|uniq|sort) +_COMPLIABLE = $(shell find src/ -name '*.go' | xargs -I{} dirname {}|sed 's/src\///g'|uniq|sort) + +GO_TESTS = \ + $(addsuffix .test,$(_TESTABLES)) + +BUILDABLES = \ + mixer.build + +include Makefile.gobuild + +# We want to add compliance for all built binaries +_CHECK_COMPLIANCE = $(addsuffix .compliant,$(_COMPLIABLE)) + +# Ensure our own code is compliant.. +compliant: $(_CHECK_COMPLIANCE) +install: $(BINS) + test -d $(DESTDIR)/usr/bin || install -D -d -m 00755 $(DESTDIR)/usr/bin; \ + install -m 00755 bin/* $(DESTDIR)/usr/bin/. + install -m 00755 pack-maker.sh $(DESTDIR)/usr/bin/mixer-pack-maker.sh + install -m 00755 superpack-maker.sh $(DESTDIR)/usr/bin/mixer-superpack-maker.sh + install -m 00644 yum.conf.in /usr/share/defaults/mixer/ + +release: + git archive --format=tar.gz --verbose -o mixer-$(VERSION).tar.gz HEAD --prefix=mixer-$(VERSION)/ + +all: compliant $(BUILDABLES) diff --git a/Makefile.gobuild b/Makefile.gobuild new file mode 100644 index 0000000..5d59876 --- /dev/null +++ b/Makefile.gobuild @@ -0,0 +1,42 @@ +CUR_DIR = $(shell pwd) + +%.build: + GOPATH=$(CUR_DIR) go install $(subst .build,,$@) + +clean: + test ! -d $(CUR_DIR)/pkg || rm -rvf $(CUR_DIR)/pkg; \ + test ! -d $(CUR_DIR)/bin || rm -rvf $(CUR_DIR)/bin + +%.compliant: + @ ( \ + cd "$(PROJECT_ROOT)/$(subst .compliant,,$@)" >/dev/null || exit 1; \ + go fmt || exit 1; \ + GOPATH=$(CUR_DIR)/ go vet || exit 1; \ + ); + +prep_coverage: + @ ( \ + echo "mode: count" > coverage.out; \ + ); + +%.test: prep_coverage + @ ( \ + safe_nom=`echo "$(subst .test,,$@)" | sed 's/\//_/g'`; \ + GOPATH=$(CUR_DIR) go test -v -cover -covermode=count -coverprofile=_coverage_$$safe_nom.out $(subst .test,,$@); \ + tail -n +2 _coverage_$$safe_nom.out >> coverage.out; \ + rm _coverage_$$safe_nom.out; \ + ); + +%.benchmark: + @ ( \ + safe_nom=`echo "$(subst .benchmark,,$@)" | sed 's/\//_/g'`; \ + cd "$(PROJECT_ROOT)/$(subst .benchmark,,$@)" >/dev/null || exit 1; \ + GOPATH=$(CUR_DIR) go test -run=XXX -v -bench=. -cpuprofile=$(CUR_DIR)/$$safe_nom.cpuprofile -memprofile=$(CUR_DIR)/$$safe_nom.memprofile; \ + ); + +check: $(GO_TESTS) + +bench: $(GO_BENCH) + +coverage: check + GOPATH=$(CUR_DIR)/ go tool cover -html=coverage.out -o coverage.html diff --git a/src/builder/builder.go b/src/builder/builder.go new file mode 100644 index 0000000..0df176a --- /dev/null +++ b/src/builder/builder.go @@ -0,0 +1,504 @@ +package builder + +import ( + "bytes" + "crypto/rsa" + "crypto/x509" + "fmt" + "io/ioutil" + "os" + "os/exec" + "reflect" + "regexp" + "strconv" + "strings" + + "helpers" +) + +// A Builder contains all configurable fields required to perform a full mix +// operation, and is used to encapsulate life time data. +type Builder struct { + Buildscript string + Buildconf string + + Bundledir string + Cert string + Clearver string + Format string + Mixver string + Repodir string + Rpmdir string + Statedir string + Versiondir string + Yumconf string + Yumtemplate string + + Signing int + Bump int +} + +// New will return a new instance of Builder with some predetermined sane +// default values. +func New() *Builder { + return &Builder{ + Buildscript: "bundle-chroot-builder.py", + Yumtemplate: "/usr/share/defaults/mixer/yum.conf.in", + + Signing: 1, + Bump: 0, + } +} + +// Get provides a useful wrapper function to pull a named field from the Builder +// instance through reflection, i.e. in assisting with parsing config files. +func (b *Builder) Get(name string) string { + return reflect.ValueOf(b).Elem().FieldByName(name).String() +} + +// CheckDeps will perform host validation to ensure that mixer has all programs +// that are required during our lifetime, available at startup. +// Missing dependencies are fatal, so we bail early to ensure we have access +// to them. +func (b *Builder) CheckDeps() bool { + deps := []string{ + "createrepo_c", + "git", + "hardlink", + "m4", + "openssl", + "parallel", + "rpm", + "yum", + } + for _, i := range deps { + if _, err := exec.LookPath(i); err != nil { + helpers.PrintError(err) + fmt.Fprintf(os.Stderr, "ERROR: Failed to find package \"%s\"\n", i) + return false + } + } + return true +} + +// LoadBuilderConf will read the builder configuration from the command line if +// it was provided, otherwise it will fall back to reading the configuration from +// the local builder.conf file. +func (b *Builder) LoadBuilderConf(builderconf string) { + local, err := os.Getwd() + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + + // If builderconf is set via cmd line, use that one + if len(builderconf) > 0 { + b.Buildconf = builderconf + return + } + + // Check if there's a local builder.conf if one wasn't supplied + localpath := local + "/builder.conf" + if _, err := os.Stat(localpath); err == nil { + b.Buildconf = localpath + } else { + helpers.PrintError(err) + fmt.Println("ERROR: Cannot find any builder.conf to use!") + os.Exit(1) + } +} + +// ReadBuilderConf will populate the configuration data from the builder +// configuration file, which is mandatory information for performing a mix. +func (b *Builder) ReadBuilderConf() { + lines, err := helpers.ReadFileAndSplit(b.Buildconf) + if err != nil { + fmt.Println("ERROR: Failed to read buildconf") + os.Exit(1) + } + + // Map the builder values to the regex here to make it easier to assign + fields := []struct { + re string + dest *string + }{ + {`^BUNDLE_DIR\s*=\s*`, &b.Bundledir}, + {`^CERT\s*=\s*`, &b.Cert}, + {`^CLEARVER\s*=\s*`, &b.Clearver}, + {`^FORMAT\s*=\s*`, &b.Format}, + {`^MIXVER\s*=\s*`, &b.Mixver}, + {`^REPODIR\s*=\s*`, &b.Repodir}, + {`^RPMDIR\s*=\s*`, &b.Rpmdir}, + {`^SERVER_STATE_DIR\s*=\s*`, &b.Statedir}, + {`^VERSIONS_PATH\s*=\s*`, &b.Versiondir}, + {`^YUM_CONF\s*=\s*`, &b.Yumconf}, + } + + for _, h := range fields { + r := regexp.MustCompile(h.re) + for _, i := range lines { + if m := r.FindIndex([]byte(i)); m != nil { + *h.dest = i[m[1]:] + } + } + } +} + +// ReadVersions will initialise the mix versions (mix and clearlinux) from +// the configuration files in the version directory. +func (b *Builder) ReadVersions() { + ver, err := ioutil.ReadFile(b.Versiondir + "/.mixversion") + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + b.Mixver = strings.TrimSpace(string(ver)) + + ver, err = ioutil.ReadFile(b.Versiondir + "/.clearversion") + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + b.Clearver = string(ver) +} + +// SignManifestMOM will sign the Manifest.Mom file in in place based on the Mix +// version read from builder.conf. +// Shelling out to openssl because signing and pkcs7 stuff is not well supported +// in Go yet.. but the command works well and is how things worked previously +func (b *Builder) SignManifestMOM() { + manifestMOM := b.Statedir + "/www/" + b.Mixver + "/Manifest.MoM" + manifestMOMsig := manifestMOM + ".sig" + cmd := exec.Command("openssl", "smime", "-sign", "-binary", "-in", manifestMOM, + "-signer", b.Cert, "-inkey", "private.pem", + "-outform", "DER", "-out", manifestMOMsig) + + // OpenSSL gives us useful info here so capture it if needed + var out bytes.Buffer + cmd.Stderr = &out + err := cmd.Run() + if err != nil { + fmt.Println("ERROR: Failed to sign Manifest.MoM!") + fmt.Printf("%s\n", out.String()) + helpers.PrintError(err) + } + fmt.Println("Signed Manifest.MoM") +} + +// UpdateRepo will fetch the clr-bundles for our configured Clear Linux version +func (b *Builder) UpdateRepo(ver string, allbundles bool) { + // Make the folder to store all clr-bundles version + if _, err := os.Stat("clr-bundles"); err != nil { + os.Mkdir("clr-bundles", 0777) + } + + repo := "clr-bundles/clr-bundles-" + ver + ".tar.gz" + if _, err := os.Stat(repo); err == nil { + fmt.Println("Already downloaded " + repo) + return + } + + URL := "https://github.com/clearlinux/clr-bundles/archive/" + ver + ".tar.gz" + err := helpers.Download(repo, URL) + if err != nil { + fmt.Println("ERROR: Failed to download new clr-bundles, make sure the version is valid") + os.Exit(1) + } + + // FIXME: Maybe use Go's tar or compress packages to do this + _, err = exec.Command("tar", "-xzf", repo, "-C", "clr-bundles/").Output() + bundles := b.Bundledir + if _, err := os.Stat(bundles); os.IsNotExist(err) { + clrbundles := "clr-bundles/clr-bundles-" + ver + "/bundles/" + os.Mkdir(bundles, 0777) + // Copy all bundles over into mix-bundles if -all passed + if allbundles == true { + files, err := ioutil.ReadDir("clr-bundles/clr-bundles-" + ver + "/bundles/") + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + for _, file := range files { + helpers.CopyFile(bundles+"/"+file.Name(), clrbundles+file.Name()) + } + } else { + // Install only a minimal set of bundles + fmt.Println("Adding os-core, os-core-update, kernel-native, bootloader to mix-bundles...") + helpers.CopyFile(bundles+"/os-core", clrbundles+"os-core") + helpers.CopyFile(bundles+"/os-core-update", clrbundles+"os-core-update") + helpers.CopyFile(bundles+"/kernel-native", clrbundles+"kernel-native") + helpers.CopyFile(bundles+"/bootloader", clrbundles+"bootloader") + } + + // Save current dir so we can get back to it + curr, err := os.Getwd() + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + os.Chdir(b.Bundledir) + helpers.GitInit() + helpers.GitAdd() + helpers.GitCommit("Initial Mix version " + b.Mixver) + os.Chdir(curr) + } + + fmt.Println("Downloaded " + repo) +} + +// InitMix will initialise a new swupd-client consumable "mix" with the given +// based Clear Linux version and specified mix version. +func (b *Builder) InitMix(clearver string, mixver string, all bool) error { + if clearver == "0" || mixver == "0" { + fmt.Println("ERROR: Please supply -clearver and -mixver") + os.Exit(1) + } + err := ioutil.WriteFile(b.Versiondir+"/.clearversion", []byte(clearver), 0644) + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + b.Mixver = mixver + + err = ioutil.WriteFile(b.Versiondir+"/.mixversion", []byte(mixver), 0644) + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + b.Clearver = clearver + + b.UpdateRepo(clearver, all) + + return nil +} + +// BuildChroots will attempt to construct the chroots required by populating roots +// using the m4 bundle configurations in conjunction with the YUM configuration file, +// installing all required named packages into the roots. +func (b *Builder) BuildChroots(template *x509.Certificate, privkey *rsa.PrivateKey, signflag bool) error { + // Generate the yum config file if it does not exist. + // This takes the template and adds the relevant local rpm repo path if needed + fmt.Println("Building chroots..") + if _, err := os.Stat(b.Yumconf); os.IsNotExist(err) { + outfile, err := os.Create(b.Yumconf) + if err != nil { + helpers.PrintError(err) + panic(err) + } + defer outfile.Close() + if b.Repodir == "" { + cmd := exec.Command("m4", b.Yumtemplate) + cmd.Stdout = outfile + cmd.Run() + + } else { + cmd := exec.Command("m4", "-D", "MIXER_REPO", "-D", "MIXER_REPOPATH="+b.Repodir, b.Yumtemplate) + cmd.Stdout = outfile + cmd.Run() + } + outfile.Close() + if err != nil { + helpers.PrintError(err) + return err + } + } + + // If MIXVER already exists, wipe it so it's a fresh build + if _, err := os.Stat(b.Statedir + "/image/" + b.Mixver); err == nil { + fmt.Printf("Wiping away previous version %s...\n", b.Mixver) + err = os.RemoveAll(b.Statedir + "/www/" + b.Mixver) + if err != nil { + return err + } + err = os.RemoveAll(b.Statedir + "/image/" + b.Mixver) + if err != nil { + return err + } + } + + // If this is a mix, we need to build with the Clear version, but publish the mix version + chrootcmd := exec.Command(b.Buildscript, "-c", b.Buildconf, "-m", b.Mixver, b.Clearver) + chrootcmd.Stdout = os.Stdout + chrootcmd.Stderr = os.Stderr + err := chrootcmd.Run() + if err != nil { + helpers.PrintError(err) + return err + } + + // Generate the certificate needed for signing verification if it does not exist and insert it into the chroot + if signflag == false && template != nil { + err = helpers.GenerateCertificate(b.Cert, template, template, &privkey.PublicKey, privkey) + if err != nil { + return err + } + } + + // Only copy the certificate into the mix if it exists + if _, err := os.Stat(b.Cert); err == nil { + chrootcert := b.Statedir + "/image/" + b.Mixver + "/os-core-update/usr/share/clear/update-ca/Swupd_Root.pem" + fmt.Println("Copying Certificate into chroot...") + err = helpers.CopyFile(chrootcert, b.Cert) + if err != nil { + helpers.PrintError(err) + return err + } + } + + // TODO: Remove all the files-* entries since they're now copied into the noship dir + // do code stuff here + + return nil +} + +// Set the published versions to what was just built +func (b *Builder) setVersion(publish bool) { + if publish == false { + return + } + + // Create the www/version/format# dir if it doesn't exist + formatdir := b.Statedir + "/www/version/format" + b.Format + if _, err := os.Stat(formatdir); os.IsNotExist(err) { + os.MkdirAll(formatdir, 0777) + } + + fmt.Println("Setting latest version to " + b.Mixver) + err := ioutil.WriteFile(formatdir+"/latest", []byte(b.Mixver), 0644) + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + + err = ioutil.WriteFile(b.Statedir+"/image/LAST_VER", []byte(b.Mixver), 0644) + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + +} + +// CleanChroots will remove chroots based on what bundles are defined +func (b *Builder) CleanChroots() { + files := helpers.GetDirContents(b.Bundledir) + basedir := b.Statedir + "/image/" + b.Mixver + "/" + + for _, f := range files { + if f.Name() == "full" { + continue + } + err := os.RemoveAll(basedir + f.Name()) + if err != nil { + helpers.PrintError(err) + } + } +} + +// BuildUpdate will produce an update consumable by the swupd client +func (b *Builder) BuildUpdate(prefixflag string, minvflag int, formatflag string, signflag bool, publishflag bool, keepchrootsflag bool) error { + if formatflag != "" { + b.Format = formatflag + } + + if _, err := os.Stat(b.Statedir + "www/version/format" + b.Format); os.IsNotExist(err) { + os.Mkdir(b.Statedir+"www/version/format"+b.Format, 0777) + } + + // Step 1: create update content for the current mix + updatecmd := exec.Command(prefixflag+"swupd_create_update", "-S", b.Statedir, "--minversion", strconv.Itoa(minvflag), "-F", b.Format, "--osversion", b.Mixver) + updatecmd.Stdout = os.Stdout + updatecmd.Stderr = os.Stderr + err := updatecmd.Run() + if err != nil { + helpers.PrintError(err) + return err + } + + // We only need the full chroot from this point on, so cleanup the others to save space + if keepchrootsflag == false { + b.CleanChroots() + } + + // Step 1.5: sign the Manifest.MoM that was just created + if signflag == false { + b.SignManifestMOM() + } + + // Step 2: create fullfiles + output, err := exec.Command(prefixflag+"swupd_make_fullfiles", "-S", b.Statedir, b.Mixver).Output() + if err != nil { + helpers.PrintError(err) + return err + } + fmt.Println(string(output)) + + // Step 3: create zero packs + if prefixflag == "" { + output, err = exec.Command("mixer-pack-maker.sh", "--to", b.Mixver, "-S", b.Statedir).Output() + } else { + output, err = exec.Command("mixer-pack-maker.sh", "--to", b.Mixver, "-S", b.Statedir, "--repodir", prefixflag).Output() + } + if err != nil { + helpers.PrintError(err) + return err + } + fmt.Println(string(output)) + + // Step 4: hardlink relevant dirs + _, err = exec.Command("hardlink", "-f", b.Statedir+"/image/"+b.Mixver+"/").Output() + + // Step 5: update the latest version + b.setVersion(publishflag) + + return nil +} + +// BuildImage will now proceed to build the full image with the previously +// validated configuration. +func (b *Builder) BuildImage(format string) { + // If the user did not pass in a format, default to builder.conf + if format == "" { + format = b.Format + } + + content := "file://" + b.Statedir + "/www" + imagecmd := exec.Command("ister.py", "-t", "release-image-config.json", "-V", content, "-C", content, "-f", format, "-s", b.Cert) + imagecmd.Stdout = os.Stdout + imagecmd.Stderr = os.Stderr + + err := imagecmd.Run() + if err != nil { + helpers.PrintError(err) + fmt.Println("Failed to create image, check /var/log/ister") + } +} + +// AddRPMList copies rpms into the repodir and calls createrepo_c on it to +// generate a yum-consumable repository for the chroot builder to use. +func (b *Builder) AddRPMList(rpms []os.FileInfo) { + for _, rpm := range rpms { + if err := helpers.CheckRPM(b.Rpmdir + "/" + rpm.Name()); err != nil { + fmt.Println("ERROR: RPM is not valid! Please make sure it was built correctly.") + os.Exit(1) + } else { + fmt.Printf("Copying %s\n", rpm.Name()) + helpers.CopyFile(b.Repodir+"/"+rpm.Name(), b.Rpmdir+"/"+rpm.Name()) + } + } + // Save current dir so we can get back to it + curr, err := os.Getwd() + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + os.Chdir(b.Repodir) + createcmd := exec.Command("createrepo_c", ".") + createcmd.Stdout = os.Stdout + createcmd.Stderr = os.Stderr + err = createcmd.Run() + if err != nil { + helpers.PrintError(err) + os.Exit(1) + } + os.Chdir(curr) +} diff --git a/src/helpers/helpers.go b/src/helpers/helpers.go new file mode 100644 index 0000000..a934c29 --- /dev/null +++ b/src/helpers/helpers.go @@ -0,0 +1,232 @@ +package helpers + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io" + "io/ioutil" + "math/big" + "net/http" + "os" + "os/exec" + "strings" + "time" +) + +var ( + // ENOVERSION is returned when an a version is unknown + ENOVERSION = 24 +) + +// PrintError is a utility function to emit an error to the console +func PrintError(e error) { + fmt.Fprintf(os.Stderr, "***Error: %v\n", e) +} + +// CreateCertTemplate will construct the template for needed openssl metadata +// instead of using an attributes.cnf file +func CreateCertTemplate() *x509.Certificate { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialnumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + fmt.Println("ERROR: Failed to generate serial number") + PrintError(err) + } + + template := x509.Certificate{ + SerialNumber: serialnumber, + Subject: pkix.Name{Organization: []string{"Mixer"}}, + SignatureAlgorithm: x509.SHA256WithRSA, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + BasicConstraintsValid: true, + IsCA: false, // This could be true since we are self signed, but set false for correctness + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature | x509.KeyUsageCRLSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageCodeSigning}, + } + + return &template +} + +// CreateKeyPair constructs an RSA keypair in memory +func CreateKeyPair() (*rsa.PrivateKey, error) { + rootKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + fmt.Println("ERROR: Failed to generate random key") + PrintError(err) + } + return rootKey, nil +} + +// GenerateCertificate will create the private signing key and public +// certificate for clients to use and writes them to disk +func GenerateCertificate(cert string, template, parent *x509.Certificate, pubkey interface{}, privkey interface{}) error { + if _, err := os.Stat(cert); os.IsNotExist(err) { + der, err := x509.CreateCertificate(rand.Reader, template, parent, pubkey, privkey) + if err != nil { + fmt.Println("ERROR: Failed to create certificate!") + PrintError(err) + return err + } + + // Write the public certficiate out for clients to use + certOut, err := os.Create("Swupd_Root.pem") + if err != nil { + fmt.Printf("failed to open cert.pem for writing: %v\n", err) + PrintError(err) + } + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: der}) + certOut.Close() + + // Write the private signing key out + keyOut, err := os.OpenFile("private.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + fmt.Println("failed to open key.pem for writing") + PrintError(err) + return err + } + // Need type assertion for Marshal to work + priv := privkey.(*rsa.PrivateKey) + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + } + + return nil +} + +// ReadFileAndSplit tokenizes the given file and converts in into a slice split +// by the newline character. +func ReadFileAndSplit(filename string) ([]string, error) { + builder, err := ioutil.ReadFile(filename) + if err != nil { + PrintError(err) + return nil, err + } + data := string(builder) + lines := strings.Split(data, "\n") + + return lines, nil +} + +// CopyFile is used during the build process to copy a given file to the target +// instead of dealing with the particulars of hardlinking. +func CopyFile(dest string, src string) error { + source, err := os.Open(src) + if err != nil { + PrintError(err) + return err + } + defer source.Close() + + destination, err := os.Create(dest) + if err != nil { + PrintError(err) + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + if err != nil { + PrintError(err) + return err + } + + err = destination.Sync() + if err != nil { + PrintError(err) + return err + } + + return nil +} + +// Download will attempt to download a from URL to the given filename +func Download(filename string, url string) (err error) { + out, err := os.Create(filename) + if err != nil { + PrintError(err) + return err + } + defer out.Close() + + infile, err := http.Get(url) + if err != nil { + PrintError(err) + return err + } + defer infile.Body.Close() + + _, err = io.Copy(out, infile.Body) + if err != nil { + PrintError(err) + return err + } + + return nil +} + +// GetDirContents is an an assert-style helper to get the contents of a +// directory, or to exit on failure. +func GetDirContents(dirname string) []os.FileInfo { + files, err := ioutil.ReadDir(dirname) + if err != nil { + PrintError(err) + os.Exit(1) + } + + return files +} + +// GitInit attempts to initialize an empty git repository in the current +// directory, or exits on failure. +func GitInit() { + gitcmd := exec.Command("git", "init") + gitcmd.Stdout = os.Stdout + err := gitcmd.Run() + if err != nil { + PrintError(err) + fmt.Println("Failed to init git repo, exiting...") + os.Exit(1) + } +} + +// GitAdd performs a 'git add .' in the current directory +func GitAdd() { + gitcmd := exec.Command("git", "add", ".") + gitcmd.Stdout = os.Stdout + err := gitcmd.Run() + if err != nil { + PrintError(err) + fmt.Println("Failed to add to git repo, exiting...") + os.Exit(1) + } +} + +// GitCommit commits to a repo with a passed in string as the commit message +func GitCommit(commitmsg string) { + gitcmd := exec.Command("git", "commit", "-m", commitmsg) + gitcmd.Stdout = os.Stdout + err := gitcmd.Run() + if err != nil { + PrintError(err) + fmt.Println("Failed to commit to git repo, exiting...") + os.Exit(1) + } +} + +// CheckRPM returns nil if file .rpm shows a valid RPM v# output, +// in order to catch corrupt or invalid RPM files. +func CheckRPM(rpm string) error { + output, err := exec.Command("file", rpm).Output() + if err != nil { + PrintError(err) + return err + } + if strings.Contains(string(output), "RPM v") { + return nil + } + return fmt.Errorf("ERROR: %s is not valid!", rpm) +} diff --git a/src/mixer/main.go b/src/mixer/main.go new file mode 100644 index 0000000..45c623c --- /dev/null +++ b/src/mixer/main.go @@ -0,0 +1,151 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "strconv" + + "builder" + "helpers" +) + +// PrintMainHelp emits useful help text to the console +func PrintMainHelp() { + fmt.Printf("usage: mixer [args]\n") + fmt.Printf("\tbuild-chroots\t\tBuild chroots for the mix\n") + fmt.Printf("\tbuild-update\t\tBuild all update content for the mix\n") + fmt.Printf("\tbuild-image\t\tBuild an image from the mix content\n") + fmt.Printf("\tadd-rpms\t\tAdd rpms to local yum repository\n") + fmt.Printf("\tget-bundles\t\tGet the clr-bundles from upstream\n") + fmt.Printf("\tinit-mix\t\tInitialize the mixer and workspace\n") +} + +// SetupBuilder performs the initial bootstrap and configuration according to +// the local configuration. +func SetupBuilder(conf string, config interface{}) { + builder := config.(*builder.Builder) + builder.LoadBuilderConf(conf) + builder.ReadBuilderConf() + builder.ReadVersions() +} + +func main() { + fmt.Println("Go Mixer 0.1") + os.Setenv("LD_PRELOAD", "/usr/lib64/nosync/nosync.so") + + addcmd := flag.NewFlagSet("add-rpms", flag.ExitOnError) + addconf := addcmd.String("config", "", "Supply a specific builder.conf to use for mixing") + + chrootcmd := flag.NewFlagSet("build-chroots", flag.ExitOnError) + certflag := chrootcmd.Bool("no-signing", false, "Do not generate a certificate to sign the Manifest.MoM") + chrootconf := chrootcmd.String("config", "", "Supply a specific builder.conf to use for mixing") + + updatecmd := flag.NewFlagSet("build-update", flag.ExitOnError) + updateconf := updatecmd.String("config", "", "Supply a specific builder.conf to use for mixing") + formatflag := updatecmd.String("format", "", "Supply format to use") + minvflag := updatecmd.Int("minversion", 0, "Supply minversion to build update with") + signflag := updatecmd.Bool("no-signing", false, "Do not generate a certificate and do not sign the Manifest.MoM") + prefixflag := updatecmd.String("prefix", "", "Supply prefix for where the swupd binaries live") + publishflag := updatecmd.Bool("no-publish", false, "Do not update the latest version after update") + keepchrootsflag := updatecmd.Bool("keep-chroots", false, "Keep individual chroots created and not just consolidated 'full'") + + bundlescmd := flag.NewFlagSet("get-bundles", flag.ExitOnError) + bundleconf := bundlescmd.String("config", "", "Supply a specific builder.conf to use for mixing") + + initcmd := flag.NewFlagSet("init-mix", flag.ExitOnError) + allflag := initcmd.Bool("all", false, "Create a mix with all Clear bundles included") + clearflag := initcmd.Int("clearver", 0, "Supply the Clear version to compose the mix from") + mixflag := initcmd.Int("mixver", 0, "Supply the Mix version to build") + initconf := initcmd.String("config", "", "Supply a specific builder.conf to use for mixing") + + imagecmd := flag.NewFlagSet("build-image", flag.ExitOnError) + imageformat := imagecmd.String("format", "", "Supply the format used for the Mix") + + if len(os.Args) == 1 { + PrintMainHelp() + return + } + + switch os.Args[1] { + case "build-chroots": + chrootcmd.Parse(os.Args[2:]) + case "build-update": + updatecmd.Parse(os.Args[2:]) + case "build-image": + imagecmd.Parse(os.Args[2:]) + case "add-rpms": + addcmd.Parse(os.Args[2:]) + case "get-bundles": + bundlescmd.Parse(os.Args[2:]) + case "init-mix": + initcmd.Parse(os.Args[2:]) + default: + fmt.Printf("%q is not valid command.\n", os.Args[1]) + os.Exit(-1) + } + + // Allocate a builder object to do all our mixing needs + builder := builder.New() + + // If we got this far, the flags are correct, so read the conf from + // the current directory or from the flag passed in + if addcmd.Parsed() { + SetupBuilder(*addconf, builder) + rpms, err := ioutil.ReadDir(builder.Rpmdir) + if err != nil { + fmt.Printf("ERROR: cannot read %s\n", builder.Rpmdir) + } + builder.AddRPMList(rpms) + } + + if bundlescmd.Parsed() { + SetupBuilder(*bundleconf, builder) + fmt.Println("Getting clr-bundles for version " + builder.Get("Clearver")) + builder.UpdateRepo(builder.Get("Clearver"), false) + } + + if initcmd.Parsed() { + builder.LoadBuilderConf(*initconf) + builder.ReadBuilderConf() + builder.InitMix(strconv.Itoa(*clearflag), strconv.Itoa(*mixflag), *allflag) + } + + if chrootcmd.Parsed() { + SetupBuilder(*chrootconf, builder) + + // Create the signing and validation key/cert + if _, err := os.Stat(builder.Get("Cert")); os.IsNotExist(err) { + fmt.Println("Generating certificate for signature validation...") + privkey, err := helpers.CreateKeyPair() + if err != nil { + os.Exit(1) + } + template := helpers.CreateCertTemplate() + + err = builder.BuildChroots(template, privkey, *certflag) + if err != nil { + os.Exit(-1) + } + } else { + err := builder.BuildChroots(nil, nil, true) + if err != nil { + os.Exit(-1) + } + } + } + + if updatecmd.Parsed() { + SetupBuilder(*updateconf, builder) + err := builder.BuildUpdate(*prefixflag, *minvflag, *formatflag, *signflag, !(*publishflag), *keepchrootsflag) + if err != nil { + os.Exit(-1) + } + } + + if imagecmd.Parsed() { + SetupBuilder("", builder) + builder.BuildImage(*imageformat) + } +}