clr-installer/controller/controller.go

// Copyright © 2019 Intel Corporation
//
// SPDX-License-Identifier: GPL-3.0-only

package controller

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/clearlinux/clr-installer/args"
	"github.com/clearlinux/clr-installer/cmd"
	"github.com/clearlinux/clr-installer/conf"
	"github.com/clearlinux/clr-installer/errors"
	"github.com/clearlinux/clr-installer/hostname"
	"github.com/clearlinux/clr-installer/isoutils"
	"github.com/clearlinux/clr-installer/keyboard"
	"github.com/clearlinux/clr-installer/language"
	"github.com/clearlinux/clr-installer/log"
	"github.com/clearlinux/clr-installer/model"
	"github.com/clearlinux/clr-installer/network"
	"github.com/clearlinux/clr-installer/progress"
	"github.com/clearlinux/clr-installer/storage"
	"github.com/clearlinux/clr-installer/swupd"
	"github.com/clearlinux/clr-installer/telemetry"
	"github.com/clearlinux/clr-installer/timezone"
	cuser "github.com/clearlinux/clr-installer/user"
	"github.com/clearlinux/clr-installer/utils"
)

var (
	// NetworkPassing is used to track if the latest network configuration
	// is passing; changes in proxy, etc.
	NetworkPassing bool
)

func sortMountPoint(bds []*storage.BlockDevice) []*storage.BlockDevice {
	sort.Slice(bds[:], func(i, j int) bool {
		return filepath.HasPrefix(bds[j].MountPoint, bds[i].MountPoint)
	})

	return bds
}

// Install is the main install controller, this is the entry point for a full
// installation
func Install(rootDir string, model *model.SystemInstall, options args.Args) error {
	var err error
	var version string
	var prg progress.Progress
	var encryptedUsed bool

	vars := map[string]string{
		"chrootDir": rootDir,
		"yamlDir":   filepath.Dir(options.ConfigFile),
	}

	for k, v := range model.Environment {
		vars[k] = v
	}

	preConfFile := log.GetPreConfFile()

	if err = model.WriteFile(preConfFile); err != nil {
		log.Error("Failed to write pre-install YAML file (%v) %q", err, preConfFile)
	}

	if model.EncryptionRequiresPassphrase() && model.CryptPass == "" {
		model.CryptPass = storage.GetPassPhrase()
		if model.CryptPass == "" {
			return errors.Errorf("Can not create encrypted file system, no passphrase")
		}
	}

	if !options.StubImage {
		if err = applyHooks("pre-install", vars, model.PreInstall); err != nil {
			return err
		}

		if model.Telemetry.Enabled {
			if err = model.Telemetry.CreateLocalTelemetryConf(); err != nil {
				return err
			}
			if model.Telemetry.URL != "" {
				if err = model.Telemetry.UpdateLocalTelemetryServer(); err != nil {
					return err
				}
			}
			if err = model.Telemetry.RestartLocalTelemetryServer(); err != nil {
				return err
			}
		}
	}

	if model.Version == 0 {
		version = utils.ClearVersion
	} else {
		version = fmt.Sprintf("%d", model.Version)
	}

	log.Debug("Clear Linux version: %s", version)

	// do we have the minimum required to install a system?
	if err = model.Validate(); err != nil {
		return err
	}

	// Using MassInstaller (non-UI) the network will not have been checked yet
	if !NetworkPassing && !options.StubImage {
		if err = ConfigureNetwork(model); err != nil {
			return err
		}
	}

	expandMe := []*storage.BlockDevice{}
	detachMe := []string{}
	removeMe := []string{}
	aliasMap := map[string]string{}

	// prepare image file, case the user has declared image alias then create
	// the image, setup the loop device, prepare the variable expansion
	for _, alias := range model.StorageAlias {
		var file string

		if alias.DeviceFile {
			continue
		}

		// create the image and add the alias name to the variable expansion list
		for _, tm := range model.TargetMedias {
			if tm.Name == fmt.Sprintf("${%s}", alias.Name) {
				if err = storage.MakeImage(tm, alias.File); err != nil {
					return err
				}

				expandMe = append(expandMe, tm)
			}
		}

		file, err = storage.SetupLoopDevice(alias.File)
		if err != nil {
			return errors.Wrap(err)
		}

		aliasMap[alias.Name] = filepath.Base(file)
		detachMe = append(detachMe, file)
		if !model.KeepImage {
			removeMe = append(removeMe, alias.File)
		}

		retry := 5

		// wait the loop device to be prepared and available with 5 retry attempts
		for {
			var ok bool

			if ok, err = utils.FileExists(file); err != nil {
				for _, file := range detachMe {
					storage.DetachLoopDevice(file)
				}

				return errors.Wrap(err)
			}

			if ok || retry == 0 {
				break
			}

			retry--
			time.Sleep(time.Second * 1)
		}
	}

	// defer detaching used loop devices
	defer func() {
		for _, file := range detachMe {
			storage.DetachLoopDevice(file)
		}
		for _, file := range removeMe {
			log.Debug("Removing raw image file: %s", file)
			if err = os.Remove(file); err != nil {
				log.Warning("Failed to remove image file: %s", file)
			}
		}
	}()

	// expand block device's name case we've detected image replacement cases
	for _, tm := range expandMe {
		tm.ExpandName(aliasMap)
	}

	mountPoints := []*storage.BlockDevice{}

	// prepare all the target block devices
	for _, curr := range model.TargetMedias {
		// based on the description given, write the partition table
		if err = curr.WritePartitionTable(model.LegacyBios, model.InstallSelected.WholeDisk); err != nil {
			return err
		}

		// prepare the blockdevice's partitions filesystem
		for _, ch := range curr.Children {
			if ch.Type == storage.BlockDeviceTypeCrypt {
				encryptedUsed = true

				if ch.FsTypeNotSwap() {
					msg := utils.Locale.Get("Mapping %s partition to an encrypted partition", ch.Name)
					prg = progress.NewLoop(msg)
					log.Info(msg)
					if err = ch.MapEncrypted(model.CryptPass); err != nil {
						return err
					}
					prg.Success()
				}
			}

			// Do not overwrite File System content for pre-existing
			if !ch.FormatPartition {
				msg := utils.Locale.Get("Skipping new file system for %s", ch.Name)
				log.Debug(msg)
				continue
			}

			msg := utils.Locale.Get("Writing %s file system to %s", ch.FsType, ch.Name)
			if ch.MountPoint != "" {
				msg = msg + fmt.Sprintf(" '%s'", ch.MountPoint)
			}
			prg = progress.NewLoop(msg)
			log.Info(msg)
			if err = ch.MakeFs(); err != nil {
				return err
			}
			prg.Success()

			// if we have a mount point set it for future mounting
			if ch.MountPoint != "" {
				mountPoints = append(mountPoints, ch)
			}
		}
	}

	// Update the target devices current labels and UUIDs
	if scanErr := storage.UpdateBlockDevices(model.TargetMedias); scanErr != nil {
		return scanErr
	}

	if options.StubImage {
		return nil
	}

	// mount all the prepared partitions
	for _, curr := range sortMountPoint(mountPoints) {
		log.Info("Mounting: %s", curr.MountPoint)

		if err = curr.Mount(rootDir); err != nil {
			return err
		}
	}

	defer func() {
		log.Info("Umounting rootDir: %s", rootDir)
		if storage.UmountAll() != nil {
			log.Warning("Failed to umount volumes")
			return
		}

		log.Info("Removing rootDir: %s", rootDir)
		if err = os.RemoveAll(rootDir); err != nil {
			log.Warning("Failed to remove rootDir: %s", rootDir)
		}
	}()

	err = storage.MountMetaFs(rootDir)
	if err != nil {
		return err
	}

	// If we are using NetworkManager add the basic bundle
	if network.IsNetworkManagerActive() {
		model.AddBundle(network.RequiredBundle)
	}

	// Add in the User Defined bundles
	for _, curr := range model.UserBundles {
		model.AddBundle(curr)
	}

	if model.Telemetry.Enabled {
		model.AddBundle(telemetry.RequiredBundle)
	}

	if len(model.Users) > 0 {
		model.AddBundle(cuser.RequiredBundle)
	}

	if model.Timezone.Code != timezone.DefaultTimezone {
		model.AddBundle(timezone.RequiredBundle)
	}

	if model.Keyboard.Code != keyboard.DefaultKeyboard {
		model.AddBundle(keyboard.RequiredBundle)
	}

	if model.Language.Code != language.DefaultLanguage {
		model.AddBundle(language.RequiredBundle)
	}

	if encryptedUsed {
		model.AddBundle(storage.RequiredBundle)
		kernelArgs := []string{storage.KernelArgument}
		model.AddExtraKernelArguments(kernelArgs)
	}

	msg := utils.Locale.Get("Writing mount files")
	prg = progress.NewLoop(msg)
	log.Info(msg)
	if err = storage.GenerateTabFiles(rootDir, model.TargetMedias); err != nil {
		return err
	}
	prg.Success()

	if model.KernelArguments != nil && len(model.KernelArguments.Add) > 0 {
		cmdlineDir := filepath.Join(rootDir, "etc", "kernel")
		cmdlineFile := filepath.Join(cmdlineDir, "cmdline")
		cmdline := strings.Join(model.KernelArguments.Add, " ")

		if err = utils.MkdirAll(cmdlineDir, 0755); err != nil {
			return err
		}

		if err = ioutil.WriteFile(cmdlineFile, []byte(cmdline), 0644); err != nil {
			return err
		}
	}

	if model.KernelArguments != nil && len(model.KernelArguments.Remove) > 0 {
		cmdlineDir := filepath.Join(rootDir, "etc", "kernel", "cmdline-removal.d")
		cmdlineFile := filepath.Join(cmdlineDir, "clr-installer.conf")
		cmdline := strings.Join(model.KernelArguments.Remove, " ")

		if err = utils.MkdirAll(cmdlineDir, 0755); err != nil {
			return err
		}

		if err = ioutil.WriteFile(cmdlineFile, []byte(cmdline), 0644); err != nil {
			return err
		}
	}

	if prg, err = contentInstall(rootDir, version, model, options); err != nil {
		prg.Failure()
		return err
	}

	if err = configureTimezone(rootDir, model); err != nil {
		// Just log the error, not setting the timezone is not reason to fail the install
		log.Error("Error setting timezone: %v", err)
	}

	if err = configureKeyboard(rootDir, model); err != nil {
		// Just log the error, not setting the keyboard is not reason to fail the install
		log.Error("Error setting keyboard: %v", err)
	}

	if err = configureLanguage(rootDir, model); err != nil {
		// Just log the error, not setting the language is not reason to fail the install
		log.Error("Error setting language locale: %v", err)
	}

	if err = cuser.Apply(rootDir, model.Users); err != nil {
		return err
	}

	if model.Hostname != "" {
		if err = hostname.SetTargetHostname(rootDir, model.Hostname); err != nil {
			return err
		}
	}

	if model.CopyNetwork {
		if err = network.CopyNetworkInterfaces(rootDir); err != nil {
			return err
		}
	}

	if model.Telemetry.URL != "" {
		if err = model.Telemetry.CreateTelemetryConf(rootDir); err != nil {
			return err
		}
	}

	if err = applyHooks("post-install", vars, model.PostInstall); err != nil {
		return err
	}

	msg = utils.Locale.Get("Saving the installation results")
	prg = progress.NewLoop(msg)
	log.Info(msg)
	if err = saveInstallResults(rootDir, model); err != nil {
		log.ErrorError(err)
	}
	prg.Success()

	if model.MakeISO {
		msg = "Generating ISO image"
		prg = progress.NewLoop(msg)
		log.Info(msg)
		if err = generateISO(rootDir, model, options); err != nil {
			log.ErrorError(err)
		}
		prg.Success()
	}

	msg = utils.Locale.Get("Installation completed")
	prg = progress.NewLoop(msg)
	log.Info(msg)
	prg.Success()

	return nil
}

func applyHooks(name string, vars map[string]string, hooks []*model.InstallHook) error {
	locName := utils.Locale.Get(name)
	msg := utils.Locale.Get("Running %s hooks", locName)
	prg := progress.MultiStep(len(hooks), msg)
	log.Info(msg)

	for idx, curr := range hooks {
		if err := runInstallHook(vars, curr); err != nil {
			prg.Failure()
			return err
		}
		prg.Partial(idx)
	}

	prg.Success()
	return nil
}

func runInstallHook(vars map[string]string, hook *model.InstallHook) error {
	args := []string{}
	vars["chrooted"] = "0"

	if hook.Chroot {
		args = append(args, []string{"chroot", vars["chrootDir"]}...)
		vars["chrooted"] = "1"
	}

	exec := utils.ExpandVariables(vars, hook.Cmd)
	args = append(args, []string{"bash", "-l", "-c", exec}...)

	if err := cmd.RunAndLogWithEnv(vars, args...); err != nil {
		return errors.Wrap(err)
	}

	return nil
}

// use the current host's version to bootstrap the sysroot, then update to the
// latest one and start adding new bundles
// for the bootstrap we use the hosts's swupd and the following operations are
// executed using the target swupd
func contentInstall(rootDir string, version string, model *model.SystemInstall, options args.Args) (progress.Progress, error) {

	sw := swupd.New(rootDir, options)

	bundles := model.Bundles

	if model.Kernel.Bundle != "none" {
		bundles = append(bundles, model.Kernel.Bundle)
	}

	if model.AutoUpdate {
		version = "latest"
	}

	msg := utils.Locale.Get("Installing base OS and configured bundles")
	prg := progress.NewLoop(msg)
	log.Info(msg)
	log.Debug("Installing bundles: %s", strings.Join(bundles, ", "))
	if err := sw.VerifyWithBundles(version, model.SwupdMirror, bundles); err != nil {
		return prg, err
	}
	prg.Success()

	if !model.AutoUpdate {
		msg := utils.Locale.Get("Disabling automatic updates")
		prg := progress.NewLoop(msg)
		log.Info(msg)
		if err := sw.DisableUpdate(); err != nil {
			warnMsg := utils.Locale.Get("Disabling automatic updates failed")
			log.Warning(warnMsg)
			return prg, err
		}
		prg.Success()
	}

	msg = utils.Locale.Get("Installing boot loader")
	prg = progress.NewLoop(msg)
	log.Info(msg)
	args := []string{
		fmt.Sprintf("%s/usr/bin/clr-boot-manager", rootDir),
		"update",
		fmt.Sprintf("--path=%s", rootDir),
	}

	err := cmd.RunAndLog(args...)
	if err != nil {
		return prg, errors.Wrap(err)
	}
	prg.Success()

	// Clean-up State Directory content
	if options.SwupdStateClean {
		msg = utils.Locale.Get("Cleaning Swupd state directory")
		prg = progress.NewLoop(msg)
		log.Info(msg)
		if err = sw.CleanUpState(); err != nil {
			log.ErrorError(err)
		}
		prg.Success()
	}

	return nil, nil
}

// ConfigureNetwork applies the model/configured network interfaces
func ConfigureNetwork(model *model.SystemInstall) error {
	prg, err := configureNetwork(model)
	if err != nil {
		prg.Success()
		NetworkPassing = false
		return err
	}

	NetworkPassing = true

	return nil
}

func configureNetwork(model *model.SystemInstall) (progress.Progress, error) {
	cmd.SetHTTPSProxy(model.HTTPSProxy)

	if len(model.NetworkInterfaces) > 0 {
		msg := "Applying network settings"
		prg := progress.NewLoop(msg)
		log.Info(msg)
		if err := network.Apply("/", model.NetworkInterfaces); err != nil {
			return prg, err
		}
		prg.Success()

		msg = utils.Locale.Get("Restarting network interfaces")
		prg = progress.NewLoop(msg)
		log.Info(msg)
		if err := network.Restart(); err != nil {
			return prg, err
		}
		prg.Success()
	}

	msg := utils.Locale.Get("Testing connectivity")
	prg := progress.NewLoop(msg)
	ok := false

	// 3 attempts to test connectivity
	for i := 0; i < 3; i++ {
		time.Sleep(2 * time.Second)

		log.Info(msg)
		if err := network.VerifyConnectivity(); err == nil {
			ok = true
			break
		}
		log.Warning("Attempt to verify connectivity failed")

		// Restart networking if we failed
		// The likely gain is restarting pacdiscovery to fix autoproxy
		if err := network.Restart(); err != nil {
			log.Warning("Network restart failed")
			ok = false
			break
		}
	}

	if !ok {
		return prg, errors.Errorf(utils.Locale.Get("Network is not working."))
	}

	prg.Success()

	return nil, nil
}

// configureTimezone applies the model/configured Timezone to the target
func configureTimezone(rootDir string, model *model.SystemInstall) error {
	if model.Timezone.Code == timezone.DefaultTimezone {
		log.Debug("Skipping setting timezone " + model.Timezone.Code)
		return nil
	}

	msg := "Setting Timezone to " + model.Timezone.Code
	prg := progress.NewLoop(msg)
	log.Info(msg)

	err := timezone.SetTargetTimezone(rootDir, model.Timezone.Code)
	if err != nil {
		prg.Failure()
		return err
	}
	prg.Success()

	return nil
}

// configureKeyboard applies the model/configured keyboard to the target
func configureKeyboard(rootDir string, model *model.SystemInstall) error {
	if model.Keyboard.Code == keyboard.DefaultKeyboard {
		log.Debug("Skipping setting keyboard " + model.Keyboard.Code)
		return nil
	}

	msg := "Setting Keyboard to " + model.Keyboard.Code
	prg := progress.NewLoop(msg)
	log.Info(msg)

	err := keyboard.SetTargetKeyboard(rootDir, model.Keyboard.Code)
	if err != nil {
		prg.Failure()
		return err
	}
	prg.Success()

	return nil
}

// configureLanguage applies the model/configured language to the target
func configureLanguage(rootDir string, model *model.SystemInstall) error {
	if model.Language.Code == language.DefaultLanguage {
		log.Debug("Skipping setting language locale " + model.Language.Code)
		return nil
	}

	msg := utils.Locale.Get("Setting Language locale to %s", model.Language.Code)
	prg := progress.NewLoop(msg)
	log.Info(msg)

	err := language.SetTargetLanguage(rootDir, model.Language.Code)
	//utils.SetLocale(model.Language.Code)
	if err != nil {
		prg.Failure()
		return err
	}
	prg.Success()

	return nil
}

// saveInstallResults saves the results of the installation process
// onto the target media
func saveInstallResults(rootDir string, md *model.SystemInstall) error {
	var err error
	errMsgs := []string{}

	// Log a sanitized YAML file with Telemetry
	var cleanModel model.SystemInstall
	// Marshal current into bytes
	confBytes, bytesErr := yaml.Marshal(md)
	if bytesErr != nil {
		log.Error("Failed to generate a copy of YAML data (%v)", bytesErr)
		errMsgs = append(errMsgs, "Failed to generate YAML file")
	}
	// Unmarshal into a copy
	if yamlErr := yaml.Unmarshal(confBytes, &cleanModel); yamlErr != nil {
		errMsgs = append(errMsgs, "Failed to duplicate YAML file")
	}
	// Sanitize the config data to remove any potential
	// Personal Information from the data set
	cleanModel.Users = nil             // Remove User Info
	cleanModel.Hostname = ""           // Remove user defined hostname
	cleanModel.HTTPSProxy = ""         // Remove user defined Proxy
	cleanModel.SwupdMirror = ""        // Remove user defined Swupd Mirror
	cleanModel.NetworkInterfaces = nil // Remove Network information

	// Remove the Serial number from the target media
	for _, bd := range cleanModel.TargetMedias {
		bd.Serial = ""
	}

	var payload string
	confBytes, bytesErr = yaml.Marshal(cleanModel)
	if bytesErr != nil {
		log.Error("Failed to generate a sanitized data (%v)", bytesErr)
		errMsgs = append(errMsgs, "Failed to generate a sanitized YAML file")
		payload = strings.Join(errMsgs, ";")
	} else {
		payload = string(confBytes[:])
	}

	if errLog := md.Telemetry.LogRecord("success", 1, payload); errLog != nil {
		log.Error("Failed to log Telemetry success record")
	}

	if md.PostArchive {
		log.Info("Saving Installation results to %s", rootDir)

		saveDir := filepath.Join(rootDir, "root")
		if err = utils.MkdirAll(saveDir, 0755); err != nil {
			// Fallback in the unlikely case we can't use root's home
			saveDir = rootDir
		}

		confFile := filepath.Join(saveDir, conf.ConfigFile)

		if err := md.WriteFile(confFile); err != nil {
			log.Error("Failed to write YAML file (%v) %q", err, confFile)
			errMsgs = append(errMsgs, "Failed to write YAML file")
		}

		logFile := filepath.Join(saveDir, conf.LogFile)

		if err := log.ArchiveLogFile(logFile); err != nil {
			errMsgs = append(errMsgs, "Failed to archive log file")
		}

	} else {
		log.Info("Skipping archiving of Installation results")
	}

	if md.IsTelemetryEnabled() {
		// Give Telemetry a chance to send before we shutdown and copy
		time.Sleep(2 * time.Second)

		if err := md.Telemetry.StopLocalTelemetryServer(); err != nil {
			log.Warning("Failed to stop image Telemetry server")
			errMsgs = append(errMsgs, "Failed to stop image Telemetry server")
		}

		log.Info("Copying telemetry records to target system.")
		if err := md.Telemetry.CopyTelemetryRecords(rootDir); err != nil {
			log.Warning("Failed to copy image Telemetry data")
			errMsgs = append(errMsgs, "Failed to copy image Telemetry data")
		}

		if len(errMsgs) > 0 {
			return errors.Errorf("%s", strings.Join(errMsgs, ";"))
		}
	} else {
		log.Info("Telemetry disabled, skipping record collection.")
	}

	return nil
}

// generateISO creates an ISO image from the just created raw image
func generateISO(rootDir string, md *model.SystemInstall, options args.Args) error {
	var err error
	msg := "Building ISO image"
	prg := progress.NewLoop(msg)
	log.Info(msg)

	if !md.LegacyBios {
		for _, alias := range md.StorageAlias {
			if err = isoutils.MakeIso(rootDir, strings.TrimSuffix(alias.File, filepath.Ext(alias.File)), md, options); err != nil {
				return err
			}
		}
	} else {
		err = fmt.Errorf("cannot create ISO images for configurations with LegacyBios enabled")
		log.ErrorError(err)
		prg.Failure()
		return err
	}

	prg.Success()
	return err
}