From 8dd0aed5e45c1f0098330eff76d2ebda2d0adb60 Mon Sep 17 00:00:00 2001 From: Tudor Marcu Date: Wed, 19 Oct 2016 16:31:01 -0700 Subject: [PATCH] Update all scripts to use new helper script The files now call functions defined in the helper script to stay more minimal and be easier to read. One very significant change to note is that all state variables and information needed for mixing will now be stored and read in the builder.conf. No longer will there be various hidden state files, with the exception of the yum-mix.conf which is autogenerated by the template still. This change simplifies the scripts calling convention and creates one centralized point to specify the configuration(s) needed. Signed-off-by: Tudor Marcu --- add-rpms.sh | 102 +++++++++++++++++++------------------- build-chroots.sh | 123 ++++++++++++++-------------------------------- create-update.sh | 89 ++++++++++----------------------- init-mix.sh | 54 +++++++++++--------- update-bundles.sh | 76 ++++++++++++++-------------- 5 files changed, 185 insertions(+), 259 deletions(-) diff --git a/add-rpms.sh b/add-rpms.sh index e8630c5..03856b7 100755 --- a/add-rpms.sh +++ b/add-rpms.sh @@ -1,77 +1,77 @@ #!/bin/bash +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e -# No defaults set for now; we haven't decided on BKMs yet for adding RPMs to a -# mix. For now, everything is done manually (create dir; add RPM(s) to it). -REPODIR= -RPMDIR= - function usage() { - echo -n "Usage: $0" - echo -n " [-h | --help]" - echo -n " [-r | --rpmdir ]" - echo -n " [-d | --repodir ]" - echo - echo - echo -n "The -r value specifies the directory containing binary RPMs," - echo - echo -n "and the -d value is the directory to use for the local repo." - echo + echo -e "Usage: $0\n" + echo -e "\t-h, --help\t\tShow this menu\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" } -while [ "$1" != "" ]; do - case "$1" in - -r | --rpmdir ) - shift; RPMDIR="$1" - ;; - -d | --repodir ) - shift; REPODIR="$1" - ;; - -h | --help ) - usage; exit 1 - ;; - * ) - usage; exit 1 - ;; - esac - shift + +while [[ $# > 0 ]] +do + key="$1" + case $key in + -c|--config) + BUILDERCONF="$2" + shift + ;; + -h | --help ) + usage; exit 1 + ;; + *) + usage; exit 1 + ;; + esac + shift done +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" + +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS + if [ ! -d "$RPMDIR" ]; then - echo "Path to directory containing must be specified with the -r option" - exit 1 + echo "Path to directory containing RPMs must be specified in builder.conf" + exit 1 fi if [ ! -d "$REPODIR" ]; then - echo "Path to local RPM repository must be specified with the -d option" - exit 1 + echo "Path to local RPM repository must be specified in builder.conf" + exit 1 fi RPMS=$(find "$RPMDIR" -type f -name *.rpm) if [ -z "$RPMS" ]; then - echo "No RPMS found. Exiting" - exit 1 + echo "No RPMS found. Exiting" + exit 1 fi set +e echo "$RPMS" | while read rpm; do - ret=$(file "$rpm") - echo "$ret" | grep --quiet ": RPM" - if [[ $? -ne 0 ]] ; then - echo "ERROR $rpm IS NOT VALID" - else - echo "Copying $rpm" - cp "$rpm" "$REPODIR" - fi -done + ret=$(file "$rpm") + echo "$ret" | grep --quiet ": RPM" + if [[ $? -ne 0 ]] ; then + echo "ERROR $rpm IS NOT VALID" + else + echo "Copying $rpm" + cp "$rpm" "$REPODIR" + fi + done set -e ( cd "$REPODIR" ; if type createrepo_c 1>/dev/null 2>&1; then createrepo_c .; else createrepo .; fi ); -# later scripts need to know to repo location -echo "$REPODIR" > "$PWD/.mixer-repopath" - exit 0 - -# vi: ts=8 sw=2 sts=2 et tw=80 diff --git a/build-chroots.sh b/build-chroots.sh index 83c971f..20dae2c 100755 --- a/build-chroots.sh +++ b/build-chroots.sh @@ -1,8 +1,9 @@ #!/bin/bash - -BUILDERSCRIPT="bundle-chroot-builder.py" -CLRVER=$(cat "$PWD/.clear-version") -MIXVER=$(cat "$PWD/.mix-version") +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers while [[ $# > 0 ]] do @@ -12,9 +13,13 @@ do BUILDERCONF="$2" shift ;; + -n|--no-signing) + SIGNING=0 + ;; -h|--help) echo -e "Usage: mixer-build-chroots.sh\n" - echo -e "\t-c, --config Supply specific builder.conf\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + echo -e "\t-n, --no-signing\tDo not generate a certificate and do not sign the Manifest.MoM" exit ;; *) @@ -25,102 +30,48 @@ do shift done -# FIXME: For now, only build chroots for the mix. In the future, when we run the -# ABI checker and build_comp to catch additional problems compared to upstream -# Clear, we will need to build vanilla Clear chroots as well. -BUILDTYPE="mix" - -# FIXME: LANG is set correctly as root user, but not otherwise. -# Desperately needs a fix in Clear... -export LANG="en_US.utf8" - -unset http_proxy -unset https_proxy - -check_dep() { - type $1 &> /dev/null - if [ $? -ne 0 ]; then - echo "$1 program not found... Unable to continue" - exit 1 - fi -} - -# Check dependencies - -# check_dep "abi-compliance-checker" -# check_dep "bundle-chroot-builder" -check_dep "cp" -check_dep "hardlink" -check_dep "m4" -check_dep "rpm" -check_dep "yum" - -if [ ! -e "$PWD/yum.conf.in" ]; then - cp /usr/share/defaults/mixer/yum.conf.in . -fi - -# Strip the trailing and leading whitespace on variables to sanitize them -function strip_whitespace { - sed 's/ *$//' | sed 's/^ *//' -} +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" # Read values from builder.conf, either supplied or default -if [[ ! -z $BUILDERCONF ]]; then - STATE_DIR=$(grep STATE_DIR "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - YUM_CONF=$(grep YUM_CONF "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - CERT=$(grep CERT "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) -elif [ -e "/etc/bundle-chroot-builder/builder.conf" ]; then - STATE_DIR=$(grep STATE_DIR "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - YUM_CONF=$(grep YUM_CONF "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - CERT=$(grep CERT "/etc/bundle-chroot-builder/builder.conf"| cut -d "=" -f2 | strip_whitespace) +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS + +# Generate the yum config file +# This takes the template and adds the relevant local rpm repo path if needed +if [ -z "$REPODIR" ] ; then + m4 "$YUM_TEMPLATE" > "$YUM_CONF" else - STATE_DIR=$(grep STATE_DIR "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - YUM_CONF=$(grep YUM_CONF "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - CERT=$(grep CERT "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) + m4 -D MIXER_REPO -D MIXER_REPOPATH="$REPODIR" "$YUM_TEMPLATE" > "$YUM_CONF" fi -if [ "$BUILDTYPE" = "clear" ]; then - m4 yum.conf.in > "$PWD/.yum-clear.conf" - BUILDVER=$CLRVER -elif [ "$BUILDTYPE" = "mix" ]; then - if [ ! -f "$PWD/.mixer-repopath" ] ; then - m4 yum.conf.in > "$PWD/.yum-mix.conf" - else - repopath=$(cat "$PWD/.mixer-repopath" | xargs realpath) - m4 -D MIXER_REPO -D MIXER_REPOPATH="$repopath" yum.conf.in > "$PWD/.yum-mix.conf" - fi - BUILDVER=$MIXVER +# If MIXVER already exists wipe it so it's a fresh build +if [ -d $STATE_DIR/image/$MIXVER ] ; then + echo -e "Wiping away previous version $MIXVER...\n" + sudo -E rm -rf "$STATE_DIR/www/$MIXVER" + sudo -E rm -rf "$STATE_DIR/image/$MIXVER" fi -# if BUILDVER already exists wipe it so it's a fresh build -if [ -d $STATE_DIR/image/$BUILDVER ] ; then - echo -e "Wiping away previous version $BUILDVER...\n" - sudo -E rm -rf "$STATE_DIR/www/$BUILDVER" - sudo -E rm -rf "$STATE_DIR/image/$BUILDVER" -fi - -# if this is a mix, need to build with the Clear version, but publish the mix version +# If this is a mix, we need to build with the Clear version, but publish the mix version if [[ ! -z $BUILDERCONF ]]; then - sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -c $BUILDERCONF -m $BUILDVER $CLRVER" + sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -c $BUILDERCONF -m $MIXVER $CLRVER" +elif [ -f $LOCALCONF ]; then + sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -c $LOCALCONF -m $MIXVER $CLRVER" else - sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -m $BUILDVER $CLRVER" + sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -m $MIXVER $CLRVER" fi # Create the certificate needed for signing verification if it does not exist, and then # insert it into the chroot -if [ ! -z $CERT ]; then - if [ ! -f $CERT ]; then - # This generates the private key and self signed certificate - openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 \ - -keyout private.pem -out $CERT \ - -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=www.example.com/DN=MixerCert" \ - -config /usr/share/defaults/mixer/certattributes.cnf - fi - echo -e "Installing certificate\n" - sudo cp $CERT "$STATE_DIR/image/$BUILDVER/os-core-update/usr/share/clear/update-ca/" +if [ $SIGNING -eq 1 ]; then + install_cert fi -# clean up the files-* entries since they are now copied into the webdir noship +# clean up the files-* entries since they are now copied into the noship dir for i in $(ls $STATE_DIR/image/$MIXVER | grep files-*); do sudo rm -f $STATE_DIR/image/$MIXVER/$i; diff --git a/create-update.sh b/create-update.sh index 9d7fb95..fa34cf1 100755 --- a/create-update.sh +++ b/create-update.sh @@ -1,4 +1,9 @@ #!/bin/bash +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e PREFIX= @@ -7,11 +12,6 @@ NOPUBLISH=0 ZEROPACKS=1 KEEP_CHROOTS=0 -# Strip the trailing and leading whitespace on variables to sanitize them -function strip_whitespace { - sed 's/ *$//' | sed 's/^ *//' -} - while [[ $# > 0 ]] do key="$1" @@ -21,7 +21,7 @@ do shift ;; -f|--format) - FORMAT="$(echo $2 | strip_whitespace)" + FORMAT="$2" shift ;; -m|--minversion) @@ -43,12 +43,12 @@ do ;; -h|--help) echo -e "Usage: mixer-create-update.sh\n" - echo -e "\t-c, --config Supply specific builder.conf\n" - echo -e "\t-f, --format Supply format to use\n" - echo -e "\t-m, --minversion supply minversion to build upate with\n" - echo -e "\t-p, --prefix Supply prefix for where the swupd binaries live\n" - echo -e "\t --no-publish Do not update the latest version after update \n" - echo -e "\t --keep-chroots Keep individual chroots created not just the consolidated 'full'" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + echo -e "\t-f, --format\t\tSupply format to use\n" + echo -e "\t-m, --minversion\tSupply minversion to build upate with\n" + echo -e "\t-p, --prefix Supply\tprefix for where the swupd binaries live\n" + echo -e "\t --no-publish\tDo not update the latest version after update\n" + echo -e "\t --keep-chroots\tKeep individual chroots created not just the consolidated 'full'" exit ;; *) @@ -59,28 +59,16 @@ do shift done -CLRVER=$(cat "$PWD/.clear-version") -MIXVER=$(cat "$PWD/.mix-version") +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" -if [ ! -z "$BUILDERCONF" ]; then - STATE_DIR=$(grep STATE_DIR "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - BUNDLE_DIR=$(grep BUNDLE_DIR "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - if [ -z "$FORMAT" ]; then - FORMAT=$(grep FORMAT "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - fi -elif [ -e "/etc/bundle-chroot-builder/builder.conf" ]; then - STATE_DIR=$(grep STATE_DIR "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - BUNDLE_DIR=$(grep BUNDLE_DIR "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - if [ -z "$FORMAT" ]; then - FORMAT=$(grep FORMAT "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - fi -else - STATE_DIR=$(grep STATE_DIR "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - BUNDLE_DIR=$(grep BUNDLE_DIR "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - if [ -z "$FORMAT"]; then - FORMAT=$(grep FORMAT "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - fi -fi +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS if [ -z "$FORMAT" ]; then FORMAT="staging" @@ -89,7 +77,7 @@ fi export BUNDLEREPO="$BUNDLE_DIR" if [ ! -d "$STATE_DIR/www/version/format$FORMAT" ]; then - sudo -E mkdir -p "$STATE_DIR/www/version/format$FORMAT/" + sudo -E mkdir -p "$STATE_DIR/www/version/format$FORMAT/" fi # step 1: create update content for current mix @@ -97,41 +85,18 @@ sudo -E "$PREFIX"swupd_create_update -S "$STATE_DIR" --minversion "$MINVERSION" # we only need the full chroot from this point on, so cleanup the others if [ "$KEEP_CHROOTS" -eq 0 ]; then - for BUNDLE in $(ls "$BUNDLEREPO" | grep -v "^full"); do - sudo rm -rf "$STATE_DIR/image/$MIXVER/$BUNDLE" - done + clean_chroots fi # step 1.5: sign the Manifest.MoM that was just created -if [ -f "ClearLinuxRoot.pem" ]; then - sudo openssl smime -sign -binary -in update/www/$MIXVER/Manifest.MoM \ - -signer ClearLinuxRoot.pem -inkey private.pem \ - -outform DER -out update/www/$MIXVER/Manifest.MoM.sig -fi +sign_manifest_mom # step 2: create fullfiles sudo -E "$PREFIX"swupd_make_fullfiles -S "$STATE_DIR" $MIXVER # step 3: create zero packs -MOM="$STATE_DIR/www/$MIXVER/Manifest.MoM" -if [ ! -e ${MOM} ]; then - error "no ${MOM}" -fi -BUNDLE_LIST=$(cat ${MOM} | awk -v V=${MIXVER} '$1 ~ /^M\./ && $3 == V { print $4 }') -# NOTE: for signing, pass the --signcontent option to swupd_make_pack. -# Signing is currently disabled until there are new test certs ready. if [ $ZEROPACKS -eq 1 ]; then - for BUNDLE in $BUNDLE_LIST; do - sudo -E "$PREFIX"swupd_make_pack -S "$STATE_DIR" 0 $MIXVER $BUNDLE & - done - - for job in $(jobs -p); do - wait ${job} - RET=$? - if [ "$RET" != "0" ]; then - error "zero pack subprocessor failed" - fi - done + sudo -E mixer-pack-maker.sh --to $MIXVER -S "$STATE_DIR" fi # step 4: hardlink relevant dirs @@ -139,8 +104,8 @@ sudo -E "hardlink" -f "$STATE_DIR/image/$MIXVER"/ # step 5: update latest version if [ $NOPUBLISH -eq 0 ]; then - sudo cp "$PWD/.mix-version" "$STATE_DIR/image/LAST_VER" - sudo cp "$PWD/.mix-version" "$STATE_DIR/www/version/format$FORMAT/latest" + sudo -E echo "$MIXVER" > "$STATE_DIR/image/LAST_VER" + sudo -E echo "$MIXVER" > "$STATE_DIR/www/version/format$FORMAT/latest" fi # step 6: archive the swupd-server logs for this mix build diff --git a/init-mix.sh b/init-mix.sh index e8bc124..6c15039 100755 --- a/init-mix.sh +++ b/init-mix.sh @@ -1,5 +1,9 @@ #!/bin/bash - +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e if [ -e /usr/lib/os-release ]; then @@ -7,7 +11,6 @@ if [ -e /usr/lib/os-release ]; then fi ALL=0 -MIXVER= while [[ $# > 0 ]] do @@ -17,23 +20,13 @@ do BUILDERCONF="$2" shift ;; - -b|--clear-version) - CLRVER="$2" - shift - ;; - -m|--mix-version) - MIXVER="$2" - shift - ;; -a|--all-bundles) ALL=1 ;; -h|--help) echo -e "Usage: mixer-init-mix.sh\n" - echo -e "\t-c, --config Supply specific builder.conf\n" - echo -e "\t-b, --clear-version Supply specific Clear version to build against\n" - echo -e "\t-m, --mix-version Supply the specific Mix version to build\n" - echo -e "\t-a, --all-bundles Create a mix with all Clear bundles included\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + echo -e "\t-a, --all-bundles\tCreate a mix with all Clear bundles included\n" exit ;; *) @@ -44,19 +37,29 @@ do shift done -if [ -z "$CLRVER" ]; then - echo -e "Please supply Clear version to use\n" - exit -fi +# Check dependencies before doing any more work +check_deps -if [ -z "$MIXVER" ]; then - MIXVER=10 -fi +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" + +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS echo -e "Creating initial update version $MIXVER\n" -mixer-init-versions.sh -m $MIXVER -c $CLRVER -mixer-update-bundles.sh +if [[ ! -z $BUILDERCONF ]]; then + mixer-update-bundles.sh -c $BUILDERCONF +elif [ -f $LOCALCONF ]; then + mixer-update-bundles.sh -c $LOCALCONF +else + mixer-update-bundles.sh +fi # Do not build the update content unless the --all-bundles flag is passed, user may want # to do additional changes to the bundles for the first version. @@ -66,12 +69,15 @@ if [ $ALL -eq 0 ]; then rm -rf * git checkout os-core os-core-update bootloader kernel-native git add . - git commit -s -m "Prune bundles for starting version $MIXVER" + git commit -s -m "Prune bundles for initial version $MIXVER" cd - else if [[ ! -z $BUILDERCONF ]]; then mixer-build-chroots.sh -c $BUILDERCONF mixer-create-update.sh -c $BUILDERCONF + elif [ -f $LOCALCONF ]; then + mixer-build-chroots.sh -c $LOCALCONF + mixer-create-update.sh -c $LOCALCONF else mixer-build-chroots.sh mixer-create-update.sh diff --git a/update-bundles.sh b/update-bundles.sh index eb55942..59cfbc1 100755 --- a/update-bundles.sh +++ b/update-bundles.sh @@ -1,48 +1,52 @@ #!/bin/bash +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e -CLRVER=$(cat "$PWD/.clear-version") -GITREPODIR="$PWD/mix-bundles" +while [[ $# > 0 ]] +do + key="$1" + case $key in + -c|--config) + BUILDERCONF="$2" + shift + ;; + -h|--help) + echo -e "Usage: mixer-update-bundles.sh\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + exit + ;; + *) + echo -e "Invalid option\n" + exit + ;; + esac + shift +done -update_repo() { - local repo="$1" - ( - if [ ! -d "$repo" ]; then - git clone https://github.com/clearlinux/"$repo.git" - cd "$repo" - else - cd "$repo" - # to force the update of clr-bundles "latest" tag - git fetch --tags - git checkout master - git pull origin master - fi - # checkout the tag relating to the clear version used to build against - git checkout tags/"$CLRVER" - set +e - local branch="${CLRVER}_mix" - git rev-parse --verify "$branch" - if [ $? -eq 0 ]; then - git checkout "$branch" - git pull - else - git checkout -b "$branch" - fi - set -e - cd .. - ) &> /dev/null - echo "$repo updated" -} +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" + +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS # Get the upstream clr-bundles -update_repo clr-bundles +update_repo "clr-bundles" # Set up mix bundle repo if it does not exist -if [ ! -d "$GITREPODIR" ]; then - echo "Creating initial $GITREPODIR" - mkdir "$GITREPODIR" - cd "$GITREPODIR" +if [ ! -d "$BUNDLE_DIR" ]; then + echo "Creating initial $BUNDLE_DIR" + mkdir "$BUNDLE_DIR" + cd "$BUNDLE_DIR" ( git init . cp ../clr-bundles/bundles/* .