diff --git a/builder/builder.go b/builder/builder.go index f1df051..d40a5ba 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -1221,6 +1221,18 @@ func (b *Builder) UpdateMixVer() error { return ioutil.WriteFile(filepath.Join(b.Config.Builder.VersionPath, b.MixVerFile), []byte(strconv.Itoa(mixVer+10)), 0644) } +// DecrementMixVer automatically bumps the mixversion file down -10 to prepare for the next build +// without requiring user intervention. This makes the flow slightly more automatable. +func (b *Builder) DecrementMixVer() error { + // Deprecate '.mixversion' --> 'mixversion' + if _, err := os.Stat(filepath.Join(b.Config.Builder.VersionPath, ".mixversion")); err == nil { + b.MixVerFile = ".mixversion" + fmt.Println("Warning: '.mixversion' has been deprecated. Please rename file to 'mixversion'") + } + mixVer, _ := strconv.Atoi(b.MixVer) + return ioutil.WriteFile(filepath.Join(b.Config.Builder.VersionPath, b.MixVerFile), []byte(strconv.Itoa(mixVer-10)), 0644) +} + // If Base == true, template will include the [main] and [clear] sections. // If Local == true, template will include the [local] section. type dnfConf struct { diff --git a/builder/format.go b/builder/format.go new file mode 100644 index 0000000..dd99b5b --- /dev/null +++ b/builder/format.go @@ -0,0 +1,132 @@ +// Copyright © 2017 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/clearlinux/mixer-tools/helpers" + "github.com/pkg/errors" +) + +// UpdateFormatVersion updates the builder.conf file with a new format version +func (b *Builder) UpdateFormatVersion(version string) error { + b.Config.Swupd.Format = version + + newver := "${1}" + b.Config.Swupd.Format + var re = regexp.MustCompile(`(FORMAT=)[0-9]*`) + + builderData, err := ioutil.ReadFile(b.BuildConf) + if err != nil { + return errors.Wrap(err, "Failed to read builder.conf") + } + + builderEdit := re.ReplaceAllString(string(builderData), newver) + + var filename string + if UseNewConfig { + filename, err = GetConfigPath("") + if err != nil { + return err + } + + return b.Config.SaveConfig(filename) + } + + builderOut := []byte(builderEdit) + if err = ioutil.WriteFile(b.BuildConf, builderOut, 0644); err != nil { + return errors.Wrap(err, "Failed to write new builder.conf") + } + + return nil +} + +// CopyFullGroupsINI copies the initial ini file which has ALL bundle definitions +func (b *Builder) CopyFullGroupsINI() error { + return helpers.CopyFile(filepath.Join(b.Config.Builder.ServerStateDir, "full_groups.ini"), filepath.Join(b.Config.Builder.ServerStateDir, "groups.ini")) +} + +// CopyTrimmedGroupsINI copies the new ini made with deleted bundles removed +func (b *Builder) CopyTrimmedGroupsINI() error { + return helpers.CopyFile(filepath.Join(b.Config.Builder.ServerStateDir, "trimmed_groups.ini"), filepath.Join(b.Config.Builder.ServerStateDir, "groups.ini")) +} + +// RevertFullGroupsINI copies back the full ini to the manifest creator accounts for deleted bundles +func (b *Builder) RevertFullGroupsINI() error { + return helpers.CopyFile(filepath.Join(b.Config.Builder.ServerStateDir, "groups.ini"), filepath.Join(b.Config.Builder.ServerStateDir, "full_groups.ini")) +} + +// RevertTrimmedGroupsINI copies back the trimmed INI so manifests are not created anymore for deleted bundles in new format +func (b *Builder) RevertTrimmedGroupsINI() error { + return helpers.CopyFile(filepath.Join(b.Config.Builder.ServerStateDir, "groups.ini"), filepath.Join(b.Config.Builder.ServerStateDir, "trimmed_groups.ini")) +} + +func (b *Builder) getLastBuildVersion() (string, error) { + var lastVer []byte + var err error + + filename := filepath.Join(b.Config.Builder.ServerStateDir, "image/LAST_VER") + // Likely the first build + if lastVer, err = ioutil.ReadFile(filename); os.IsNotExist(err) { + return "", nil + } else if err != nil { + return "", errors.Wrap(err, "Cannot find last built version") + } + data := string(lastVer) + ver := strings.Split(data, "\n") + + return ver[0], nil +} + +// CheckBumpNeeded returns nil if it successfully deduces there is no format +// bump boundary being crossed. +func (b *Builder) CheckBumpNeeded() (bool, error) { + version, err := b.getLastBuildVersion() + if err != nil { + return false, err + } + // Check what format our last built version is part of + oldVer, err := b.DownloadFileFromUpstream(filepath.Join("/update", version, "format")) + if err != nil { + return false, errors.Wrapf(err, "Could not read format version from %s", b.UpstreamURL) + } + // Check what format our to-be-built version is part of + newVer, err := b.DownloadFileFromUpstream(filepath.Join("/update", b.MixVer, "format")) + if err != nil { + return false, errors.Wrapf(err, "Could not read format version from %s", b.UpstreamURL) + } + + // Check both formats are real numbers + oldFmt, err := strconv.ParseUint(string(oldVer), 10, 32) + if err != nil { + return false, errors.New("Old format is not a number") + } + newFmt, err := strconv.ParseUint(string(newVer), 10, 32) + if err != nil { + return false, errors.New("Old format is not a number") + } + + // We always need to perform a format bump if these are not equal + if oldFmt != newFmt { + return true, nil + } + + return false, nil +} diff --git a/mixer/cmd/build.go b/mixer/cmd/build.go index b512d52..cb80bdc 100644 --- a/mixer/cmd/build.go +++ b/mixer/cmd/build.go @@ -17,7 +17,9 @@ package cmd import ( "fmt" "os" + "path/filepath" "runtime" + "strconv" "github.com/clearlinux/mixer-tools/builder" "github.com/clearlinux/mixer-tools/helpers" @@ -109,6 +111,151 @@ var buildBundlesCmd = &cobra.Command{ }, } +// buildOldFormatCmd is used to build the final version in the current format (the +10) +// and ready the mix state for the new (+20) version to be built by a (possibly) newer mixer +var buildOldFormatCmd = &cobra.Command{ + Use: "format-old", + Short: "Build the last version in the current format for the format bump", + Long: `Build the last version in the current format for the format bump`, + Run: func(cmd *cobra.Command, args []string) { + b, err := builder.NewFromConfig(config) + if err != nil { + fail(err) + } + + // Update the mixversion just in case the user did not pass --increment + // This must be the +20 to write the new format data files even though we + // will build a +10 from the same content + for i := 0; i < 2; i++ { + if err = b.UpdateMixVer(); err != nil { + failf("Couldn't update Mix Version") + } + // Must re-read the builder or UpdateMixVer() will be a noop each iteration + if err = b.ReadVersions(); err != nil { + fail(err) + } + } + + // Set format to format+1 so that the format file inserted into the + // update content is the new one + if err = b.UpdateFormatVersion(buildFlags.format); err != nil { + fail(err) + } + setWorkers(b) + + fmt.Println(" Backing up full groups.ini") + // Back up groups.ini in case we have deprecated bundles to delete + if err = b.CopyFullGroupsINI(); err != nil { + fail(err) + } + // Fill this in w/Update bundle definitions + // if err := UpdateBudlesForFormatBump(); err != nil {...} + + // Build the +20 (first build in new format) bundles + if err = buildBundles(b, buildFlags.noSigning); err != nil { + fail(err) + } + + fmt.Println("Backing up trimmed groups.ini for future build") + // Copy the new trimmed groups.ini to use for the +20 build which will + // not have manifest created for fully deleted bundles + if err = b.CopyTrimmedGroupsINI(); err != nil { + fail(err) + } + + // Copy +20 chroots to +10 so we can build last formatN build with the + // same content + prevVersion, err := strconv.Atoi(b.MixVer) + if err != nil { + fail(err) + } + prevVersion -= 10 + source := filepath.Join(b.Config.Builder.ServerStateDir, "image", b.MixVer) + dest := filepath.Join(b.Config.Builder.ServerStateDir, "image", strconv.Itoa(prevVersion)) + fmt.Println(" Copying +20 chroots to +10 chroots") + if err = helpers.RunCommandSilent("cp", "-al", source, dest); err != nil { + failf("Failed to copy +20 chroots to +10: %s\n", err) + } + + // Copy the old groups.ini file back which contains ALL original bundle names + // to account for any removed bundles in this build when creating manifests + fmt.Println(" Copying full groups.ini back to working directory") + if err = b.RevertFullGroupsINI(); err != nil { + fail(err) + } + + // Set the format back to the previous format version before building the +10 update + prevFormat, err := strconv.Atoi(buildFlags.format) + if err != nil { + fail(err) + } + prevFormat-- + if err = b.UpdateFormatVersion(strconv.Itoa(prevFormat)); err != nil { + fail(err) + } + // Set mixversion to the +10 since we have used +20 up to this point + if err = b.DecrementMixVer(); err != nil { + fail(err) + } + // Re-read builder.conf after updating format/mixver + if err = b.ReadVersions(); err != nil { + fail(err) + } + setWorkers(b) + + // Build the update content for the +10 build + err = b.BuildUpdate(buildFlags.prefix, buildFlags.minVersion, buildFlags.format, buildFlags.noSigning, !buildFlags.noPublish, buildFlags.keepChroot) + if err != nil { + failf("Couldn't build update: %s", err) + } + // Copy the trimmed groups.ini back for the +20 build to use + if err = b.RevertTrimmedGroupsINI(); err != nil { + fail(err) + } + // Update the format back to the formatN+1 version for +20 to use + if err = b.UpdateFormatVersion(buildFlags.format); err != nil { + fail(err) + } + // Set the previous format latest file to the +10 build + formatDir := filepath.Join(b.Config.Builder.ServerStateDir, "www", "version", "format"+strconv.Itoa(prevFormat)) + err = ioutil.WriteFile(filepath.Join(formatDir, "latest"), []byte(b.MixVer), 0644) + if err != nil { + failf("Couldn't update the latest version") + } + // Incremement regardless of flag because +20 must be the next version + if err = b.UpdateMixVer(); err != nil { + failf("Couldn't update Mix Version") + } + }, +} + +var buildNewFormatCmd = &cobra.Command{ + Use: "format-new", + Short: "Build the first version in the new format for the format bump", + Long: `Build the first version in the new format for the format bump`, + Run: func(cmd *cobra.Command, args []string) { + b, err := builder.NewFromConfig(config) + if err != nil { + fail(err) + } + setWorkers(b) + ver, err := strconv.Atoi(b.MixVer) + if err != nil { + fail(err) + } + err = b.BuildUpdate(buildFlags.prefix, ver, buildFlags.format, buildFlags.noSigning, !buildFlags.noPublish, buildFlags.keepChroot) + if err != nil { + failf("Couldn't build update: %s", err) + } + + if buildFlags.increment { + if err = b.UpdateMixVer(); err != nil { + failf("Couldn't update Mix Version") + } + } + }, +} + var buildUpdateCmd = &cobra.Command{ Use: "update", Short: "Build the update content for your mix", @@ -121,7 +268,7 @@ var buildUpdateCmd = &cobra.Command{ setWorkers(b) err = b.BuildUpdate(buildFlags.prefix, buildFlags.minVersion, buildFlags.format, buildFlags.noSigning, !buildFlags.noPublish, buildFlags.keepChroot) if err != nil { - failf("couldn't build update: %s", err) + failf("Couldn't build update: %s", err) } if buildFlags.increment { @@ -146,16 +293,16 @@ var buildAllCmd = &cobra.Command{ if err == nil { err = b.AddRPMList(rpms) if err != nil { - failf("couldn't add the RPMs: %s", err) + failf("Couldn't add the RPMs: %s", err) } } err = buildBundles(b, buildFlags.noSigning) if err != nil { - failf("couldn't build bundles: %s", err) + failf("Couldn't build bundles: %s", err) } err = b.BuildUpdate(buildFlags.prefix, buildFlags.minVersion, buildFlags.format, buildFlags.noSigning, !buildFlags.noPublish, buildFlags.keepChroot) if err != nil { - failf("couldn't build update: %s", err) + failf("Couldn't build update: %s", err) } err = b.UpdateMixVer() if err != nil { @@ -176,7 +323,7 @@ var buildImageCmd = &cobra.Command{ setWorkers(b) err = b.BuildImage(buildFlags.format, buildFlags.template) if err != nil { - failf("couldn't build image: %s", err) + failf("Couldn't build image: %s", err) } }, } @@ -255,6 +402,8 @@ var buildCmds = []*cobra.Command{ buildAllCmd, buildImageCmd, buildDeltaPacksCmd, + buildOldFormatCmd, + buildNewFormatCmd, } func init() { @@ -284,6 +433,8 @@ func init() { setUpdateFlags(buildUpdateCmd) setUpdateFlags(buildAllCmd) + setUpdateFlags(buildOldFormatCmd) + setUpdateFlags(buildNewFormatCmd) externalDeps[buildBundlesCmd] = []string{ "rpm",