diff --git a/add-rpms.sh b/add-rpms.sh index e8630c5..03856b7 100755 --- a/add-rpms.sh +++ b/add-rpms.sh @@ -1,77 +1,77 @@ #!/bin/bash +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e -# No defaults set for now; we haven't decided on BKMs yet for adding RPMs to a -# mix. For now, everything is done manually (create dir; add RPM(s) to it). -REPODIR= -RPMDIR= - function usage() { - echo -n "Usage: $0" - echo -n " [-h | --help]" - echo -n " [-r | --rpmdir ]" - echo -n " [-d | --repodir ]" - echo - echo - echo -n "The -r value specifies the directory containing binary RPMs," - echo - echo -n "and the -d value is the directory to use for the local repo." - echo + echo -e "Usage: $0\n" + echo -e "\t-h, --help\t\tShow this menu\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" } -while [ "$1" != "" ]; do - case "$1" in - -r | --rpmdir ) - shift; RPMDIR="$1" - ;; - -d | --repodir ) - shift; REPODIR="$1" - ;; - -h | --help ) - usage; exit 1 - ;; - * ) - usage; exit 1 - ;; - esac - shift + +while [[ $# > 0 ]] +do + key="$1" + case $key in + -c|--config) + BUILDERCONF="$2" + shift + ;; + -h | --help ) + usage; exit 1 + ;; + *) + usage; exit 1 + ;; + esac + shift done +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" + +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS + if [ ! -d "$RPMDIR" ]; then - echo "Path to directory containing must be specified with the -r option" - exit 1 + echo "Path to directory containing RPMs must be specified in builder.conf" + exit 1 fi if [ ! -d "$REPODIR" ]; then - echo "Path to local RPM repository must be specified with the -d option" - exit 1 + echo "Path to local RPM repository must be specified in builder.conf" + exit 1 fi RPMS=$(find "$RPMDIR" -type f -name *.rpm) if [ -z "$RPMS" ]; then - echo "No RPMS found. Exiting" - exit 1 + echo "No RPMS found. Exiting" + exit 1 fi set +e echo "$RPMS" | while read rpm; do - ret=$(file "$rpm") - echo "$ret" | grep --quiet ": RPM" - if [[ $? -ne 0 ]] ; then - echo "ERROR $rpm IS NOT VALID" - else - echo "Copying $rpm" - cp "$rpm" "$REPODIR" - fi -done + ret=$(file "$rpm") + echo "$ret" | grep --quiet ": RPM" + if [[ $? -ne 0 ]] ; then + echo "ERROR $rpm IS NOT VALID" + else + echo "Copying $rpm" + cp "$rpm" "$REPODIR" + fi + done set -e ( cd "$REPODIR" ; if type createrepo_c 1>/dev/null 2>&1; then createrepo_c .; else createrepo .; fi ); -# later scripts need to know to repo location -echo "$REPODIR" > "$PWD/.mixer-repopath" - exit 0 - -# vi: ts=8 sw=2 sts=2 et tw=80 diff --git a/build-chroots.sh b/build-chroots.sh index 83c971f..20dae2c 100755 --- a/build-chroots.sh +++ b/build-chroots.sh @@ -1,8 +1,9 @@ #!/bin/bash - -BUILDERSCRIPT="bundle-chroot-builder.py" -CLRVER=$(cat "$PWD/.clear-version") -MIXVER=$(cat "$PWD/.mix-version") +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers while [[ $# > 0 ]] do @@ -12,9 +13,13 @@ do BUILDERCONF="$2" shift ;; + -n|--no-signing) + SIGNING=0 + ;; -h|--help) echo -e "Usage: mixer-build-chroots.sh\n" - echo -e "\t-c, --config Supply specific builder.conf\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + echo -e "\t-n, --no-signing\tDo not generate a certificate and do not sign the Manifest.MoM" exit ;; *) @@ -25,102 +30,48 @@ do shift done -# FIXME: For now, only build chroots for the mix. In the future, when we run the -# ABI checker and build_comp to catch additional problems compared to upstream -# Clear, we will need to build vanilla Clear chroots as well. -BUILDTYPE="mix" - -# FIXME: LANG is set correctly as root user, but not otherwise. -# Desperately needs a fix in Clear... -export LANG="en_US.utf8" - -unset http_proxy -unset https_proxy - -check_dep() { - type $1 &> /dev/null - if [ $? -ne 0 ]; then - echo "$1 program not found... Unable to continue" - exit 1 - fi -} - -# Check dependencies - -# check_dep "abi-compliance-checker" -# check_dep "bundle-chroot-builder" -check_dep "cp" -check_dep "hardlink" -check_dep "m4" -check_dep "rpm" -check_dep "yum" - -if [ ! -e "$PWD/yum.conf.in" ]; then - cp /usr/share/defaults/mixer/yum.conf.in . -fi - -# Strip the trailing and leading whitespace on variables to sanitize them -function strip_whitespace { - sed 's/ *$//' | sed 's/^ *//' -} +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" # Read values from builder.conf, either supplied or default -if [[ ! -z $BUILDERCONF ]]; then - STATE_DIR=$(grep STATE_DIR "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - YUM_CONF=$(grep YUM_CONF "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - CERT=$(grep CERT "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) -elif [ -e "/etc/bundle-chroot-builder/builder.conf" ]; then - STATE_DIR=$(grep STATE_DIR "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - YUM_CONF=$(grep YUM_CONF "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - CERT=$(grep CERT "/etc/bundle-chroot-builder/builder.conf"| cut -d "=" -f2 | strip_whitespace) +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS + +# Generate the yum config file +# This takes the template and adds the relevant local rpm repo path if needed +if [ -z "$REPODIR" ] ; then + m4 "$YUM_TEMPLATE" > "$YUM_CONF" else - STATE_DIR=$(grep STATE_DIR "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - YUM_CONF=$(grep YUM_CONF "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - CERT=$(grep CERT "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) + m4 -D MIXER_REPO -D MIXER_REPOPATH="$REPODIR" "$YUM_TEMPLATE" > "$YUM_CONF" fi -if [ "$BUILDTYPE" = "clear" ]; then - m4 yum.conf.in > "$PWD/.yum-clear.conf" - BUILDVER=$CLRVER -elif [ "$BUILDTYPE" = "mix" ]; then - if [ ! -f "$PWD/.mixer-repopath" ] ; then - m4 yum.conf.in > "$PWD/.yum-mix.conf" - else - repopath=$(cat "$PWD/.mixer-repopath" | xargs realpath) - m4 -D MIXER_REPO -D MIXER_REPOPATH="$repopath" yum.conf.in > "$PWD/.yum-mix.conf" - fi - BUILDVER=$MIXVER +# If MIXVER already exists wipe it so it's a fresh build +if [ -d $STATE_DIR/image/$MIXVER ] ; then + echo -e "Wiping away previous version $MIXVER...\n" + sudo -E rm -rf "$STATE_DIR/www/$MIXVER" + sudo -E rm -rf "$STATE_DIR/image/$MIXVER" fi -# if BUILDVER already exists wipe it so it's a fresh build -if [ -d $STATE_DIR/image/$BUILDVER ] ; then - echo -e "Wiping away previous version $BUILDVER...\n" - sudo -E rm -rf "$STATE_DIR/www/$BUILDVER" - sudo -E rm -rf "$STATE_DIR/image/$BUILDVER" -fi - -# if this is a mix, need to build with the Clear version, but publish the mix version +# If this is a mix, we need to build with the Clear version, but publish the mix version if [[ ! -z $BUILDERCONF ]]; then - sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -c $BUILDERCONF -m $BUILDVER $CLRVER" + sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -c $BUILDERCONF -m $MIXVER $CLRVER" +elif [ -f $LOCALCONF ]; then + sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -c $LOCALCONF -m $MIXVER $CLRVER" else - sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -m $BUILDVER $CLRVER" + sudo -E sh -c "LD_PRELOAD=/usr/lib64/nosync/nosync.so $BUILDERSCRIPT -m $MIXVER $CLRVER" fi # Create the certificate needed for signing verification if it does not exist, and then # insert it into the chroot -if [ ! -z $CERT ]; then - if [ ! -f $CERT ]; then - # This generates the private key and self signed certificate - openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 \ - -keyout private.pem -out $CERT \ - -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=www.example.com/DN=MixerCert" \ - -config /usr/share/defaults/mixer/certattributes.cnf - fi - echo -e "Installing certificate\n" - sudo cp $CERT "$STATE_DIR/image/$BUILDVER/os-core-update/usr/share/clear/update-ca/" +if [ $SIGNING -eq 1 ]; then + install_cert fi -# clean up the files-* entries since they are now copied into the webdir noship +# clean up the files-* entries since they are now copied into the noship dir for i in $(ls $STATE_DIR/image/$MIXVER | grep files-*); do sudo rm -f $STATE_DIR/image/$MIXVER/$i; diff --git a/create-update.sh b/create-update.sh index 9d7fb95..fa34cf1 100755 --- a/create-update.sh +++ b/create-update.sh @@ -1,4 +1,9 @@ #!/bin/bash +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e PREFIX= @@ -7,11 +12,6 @@ NOPUBLISH=0 ZEROPACKS=1 KEEP_CHROOTS=0 -# Strip the trailing and leading whitespace on variables to sanitize them -function strip_whitespace { - sed 's/ *$//' | sed 's/^ *//' -} - while [[ $# > 0 ]] do key="$1" @@ -21,7 +21,7 @@ do shift ;; -f|--format) - FORMAT="$(echo $2 | strip_whitespace)" + FORMAT="$2" shift ;; -m|--minversion) @@ -43,12 +43,12 @@ do ;; -h|--help) echo -e "Usage: mixer-create-update.sh\n" - echo -e "\t-c, --config Supply specific builder.conf\n" - echo -e "\t-f, --format Supply format to use\n" - echo -e "\t-m, --minversion supply minversion to build upate with\n" - echo -e "\t-p, --prefix Supply prefix for where the swupd binaries live\n" - echo -e "\t --no-publish Do not update the latest version after update \n" - echo -e "\t --keep-chroots Keep individual chroots created not just the consolidated 'full'" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + echo -e "\t-f, --format\t\tSupply format to use\n" + echo -e "\t-m, --minversion\tSupply minversion to build upate with\n" + echo -e "\t-p, --prefix Supply\tprefix for where the swupd binaries live\n" + echo -e "\t --no-publish\tDo not update the latest version after update\n" + echo -e "\t --keep-chroots\tKeep individual chroots created not just the consolidated 'full'" exit ;; *) @@ -59,28 +59,16 @@ do shift done -CLRVER=$(cat "$PWD/.clear-version") -MIXVER=$(cat "$PWD/.mix-version") +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" -if [ ! -z "$BUILDERCONF" ]; then - STATE_DIR=$(grep STATE_DIR "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - BUNDLE_DIR=$(grep BUNDLE_DIR "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - if [ -z "$FORMAT" ]; then - FORMAT=$(grep FORMAT "$BUILDERCONF" | cut -d "=" -f2 | strip_whitespace) - fi -elif [ -e "/etc/bundle-chroot-builder/builder.conf" ]; then - STATE_DIR=$(grep STATE_DIR "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - BUNDLE_DIR=$(grep BUNDLE_DIR "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - if [ -z "$FORMAT" ]; then - FORMAT=$(grep FORMAT "/etc/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - fi -else - STATE_DIR=$(grep STATE_DIR "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - BUNDLE_DIR=$(grep BUNDLE_DIR "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - if [ -z "$FORMAT"]; then - FORMAT=$(grep FORMAT "/usr/share/defaults/bundle-chroot-builder/builder.conf" | cut -d "=" -f2 | strip_whitespace) - fi -fi +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS if [ -z "$FORMAT" ]; then FORMAT="staging" @@ -89,7 +77,7 @@ fi export BUNDLEREPO="$BUNDLE_DIR" if [ ! -d "$STATE_DIR/www/version/format$FORMAT" ]; then - sudo -E mkdir -p "$STATE_DIR/www/version/format$FORMAT/" + sudo -E mkdir -p "$STATE_DIR/www/version/format$FORMAT/" fi # step 1: create update content for current mix @@ -97,41 +85,18 @@ sudo -E "$PREFIX"swupd_create_update -S "$STATE_DIR" --minversion "$MINVERSION" # we only need the full chroot from this point on, so cleanup the others if [ "$KEEP_CHROOTS" -eq 0 ]; then - for BUNDLE in $(ls "$BUNDLEREPO" | grep -v "^full"); do - sudo rm -rf "$STATE_DIR/image/$MIXVER/$BUNDLE" - done + clean_chroots fi # step 1.5: sign the Manifest.MoM that was just created -if [ -f "ClearLinuxRoot.pem" ]; then - sudo openssl smime -sign -binary -in update/www/$MIXVER/Manifest.MoM \ - -signer ClearLinuxRoot.pem -inkey private.pem \ - -outform DER -out update/www/$MIXVER/Manifest.MoM.sig -fi +sign_manifest_mom # step 2: create fullfiles sudo -E "$PREFIX"swupd_make_fullfiles -S "$STATE_DIR" $MIXVER # step 3: create zero packs -MOM="$STATE_DIR/www/$MIXVER/Manifest.MoM" -if [ ! -e ${MOM} ]; then - error "no ${MOM}" -fi -BUNDLE_LIST=$(cat ${MOM} | awk -v V=${MIXVER} '$1 ~ /^M\./ && $3 == V { print $4 }') -# NOTE: for signing, pass the --signcontent option to swupd_make_pack. -# Signing is currently disabled until there are new test certs ready. if [ $ZEROPACKS -eq 1 ]; then - for BUNDLE in $BUNDLE_LIST; do - sudo -E "$PREFIX"swupd_make_pack -S "$STATE_DIR" 0 $MIXVER $BUNDLE & - done - - for job in $(jobs -p); do - wait ${job} - RET=$? - if [ "$RET" != "0" ]; then - error "zero pack subprocessor failed" - fi - done + sudo -E mixer-pack-maker.sh --to $MIXVER -S "$STATE_DIR" fi # step 4: hardlink relevant dirs @@ -139,8 +104,8 @@ sudo -E "hardlink" -f "$STATE_DIR/image/$MIXVER"/ # step 5: update latest version if [ $NOPUBLISH -eq 0 ]; then - sudo cp "$PWD/.mix-version" "$STATE_DIR/image/LAST_VER" - sudo cp "$PWD/.mix-version" "$STATE_DIR/www/version/format$FORMAT/latest" + sudo -E echo "$MIXVER" > "$STATE_DIR/image/LAST_VER" + sudo -E echo "$MIXVER" > "$STATE_DIR/www/version/format$FORMAT/latest" fi # step 6: archive the swupd-server logs for this mix build diff --git a/init-mix.sh b/init-mix.sh index e8bc124..6c15039 100755 --- a/init-mix.sh +++ b/init-mix.sh @@ -1,5 +1,9 @@ #!/bin/bash - +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e if [ -e /usr/lib/os-release ]; then @@ -7,7 +11,6 @@ if [ -e /usr/lib/os-release ]; then fi ALL=0 -MIXVER= while [[ $# > 0 ]] do @@ -17,23 +20,13 @@ do BUILDERCONF="$2" shift ;; - -b|--clear-version) - CLRVER="$2" - shift - ;; - -m|--mix-version) - MIXVER="$2" - shift - ;; -a|--all-bundles) ALL=1 ;; -h|--help) echo -e "Usage: mixer-init-mix.sh\n" - echo -e "\t-c, --config Supply specific builder.conf\n" - echo -e "\t-b, --clear-version Supply specific Clear version to build against\n" - echo -e "\t-m, --mix-version Supply the specific Mix version to build\n" - echo -e "\t-a, --all-bundles Create a mix with all Clear bundles included\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + echo -e "\t-a, --all-bundles\tCreate a mix with all Clear bundles included\n" exit ;; *) @@ -44,19 +37,29 @@ do shift done -if [ -z "$CLRVER" ]; then - echo -e "Please supply Clear version to use\n" - exit -fi +# Check dependencies before doing any more work +check_deps -if [ -z "$MIXVER" ]; then - MIXVER=10 -fi +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" + +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS echo -e "Creating initial update version $MIXVER\n" -mixer-init-versions.sh -m $MIXVER -c $CLRVER -mixer-update-bundles.sh +if [[ ! -z $BUILDERCONF ]]; then + mixer-update-bundles.sh -c $BUILDERCONF +elif [ -f $LOCALCONF ]; then + mixer-update-bundles.sh -c $LOCALCONF +else + mixer-update-bundles.sh +fi # Do not build the update content unless the --all-bundles flag is passed, user may want # to do additional changes to the bundles for the first version. @@ -66,12 +69,15 @@ if [ $ALL -eq 0 ]; then rm -rf * git checkout os-core os-core-update bootloader kernel-native git add . - git commit -s -m "Prune bundles for starting version $MIXVER" + git commit -s -m "Prune bundles for initial version $MIXVER" cd - else if [[ ! -z $BUILDERCONF ]]; then mixer-build-chroots.sh -c $BUILDERCONF mixer-create-update.sh -c $BUILDERCONF + elif [ -f $LOCALCONF ]; then + mixer-build-chroots.sh -c $LOCALCONF + mixer-create-update.sh -c $LOCALCONF else mixer-build-chroots.sh mixer-create-update.sh diff --git a/update-bundles.sh b/update-bundles.sh index eb55942..59cfbc1 100755 --- a/update-bundles.sh +++ b/update-bundles.sh @@ -1,48 +1,52 @@ #!/bin/bash +if [ ! -f /usr/share/mixer-tools/helpers ]; then + echo "Cannot find /usr/share/mixer-tools/helpers, please install first, exiting..." + exit +fi +source /usr/share/mixer-tools/helpers set -e -CLRVER=$(cat "$PWD/.clear-version") -GITREPODIR="$PWD/mix-bundles" +while [[ $# > 0 ]] +do + key="$1" + case $key in + -c|--config) + BUILDERCONF="$2" + shift + ;; + -h|--help) + echo -e "Usage: mixer-update-bundles.sh\n" + echo -e "\t-c, --config\t\tSupply specific builder.conf\n" + exit + ;; + *) + echo -e "Invalid option\n" + exit + ;; + esac + shift +done -update_repo() { - local repo="$1" - ( - if [ ! -d "$repo" ]; then - git clone https://github.com/clearlinux/"$repo.git" - cd "$repo" - else - cd "$repo" - # to force the update of clr-bundles "latest" tag - git fetch --tags - git checkout master - git pull origin master - fi - # checkout the tag relating to the clear version used to build against - git checkout tags/"$CLRVER" - set +e - local branch="${CLRVER}_mix" - git rev-parse --verify "$branch" - if [ $? -eq 0 ]; then - git checkout "$branch" - git pull - else - git checkout -b "$branch" - fi - set -e - cd .. - ) &> /dev/null - echo "$repo updated" -} +# Set the possible builder.conf files to read from +load_builder_conf +BUILDERCONFS=" +$BUILDERCONF +$LOCALCONF +" + +# Read values from builder.conf, either supplied or default +# This will prioritize reading from cmd line, etc, and then /usr/share/defaults/ +read_builder_conf $BUILDERCONFS # Get the upstream clr-bundles -update_repo clr-bundles +update_repo "clr-bundles" # Set up mix bundle repo if it does not exist -if [ ! -d "$GITREPODIR" ]; then - echo "Creating initial $GITREPODIR" - mkdir "$GITREPODIR" - cd "$GITREPODIR" +if [ ! -d "$BUNDLE_DIR" ]; then + echo "Creating initial $BUNDLE_DIR" + mkdir "$BUNDLE_DIR" + cd "$BUNDLE_DIR" ( git init . cp ../clr-bundles/bundles/* .