diff --git a/pack-maker.sh b/pack-maker.sh index d325dd4..3546cb0 100755 --- a/pack-maker.sh +++ b/pack-maker.sh @@ -1,110 +1,206 @@ -#!/bin/bash -e -# usage: -# mixer-pack-maker.sh -# mixer-pack-maker.sh 2820 2 -# mixer-pack-maker.sh 2730 3 +#!/bin/bash +# +# Usage: +# mixer-pack-maker.sh --to VER [OPTIONS] +# +# Create zero packs and/or delta packs for a given swupd build VER. +# +# The presence of the --from option enables delta pack creation from the given +# version. More than one --from option may be specified. If both zero and delta +# packs should be created at the same time, pass the --zero option in addition +# to the --from option. +# +# To force recreation of all packs (zero and/or delta), use the --force option. +# +# Example invocations: +# +# 1) Create zero packs for build 7000 +# +# mixer-pack-maker.sh --to 7000 +# +# 2) Create delta packs from build 6000 to 7000, and from 6500 to 7000 +# +# mixer-pack-maker.sh --to 7000 --from 6000 --from 6500 +# +# 3) Create delta packs from build 6900 to 7000, and zero packs for build 7000 +# +# mixer-pack-maker.sh --to 7000 --from 6900 --zero # -# For each bundle changed as of the Manifest.MoM at the , -# create version-pair packs going back -# versions of that bundle. For example if bundle "editors" changed in 2650 -# and the script is run as "pack_maker.sh 2650 2", it will find that -# "editors" last two change points were 2550 and 2260 and thus create a -# 2550-to-2650 and a 2260-to-2650 version pair pack for the "editors" -# bundle. -#NOTE: fullfiles must be created before calling this script -#NOTE: zero packs need similarly created +FROM_VERS="" +REPO_DIR="" +STATE_DIR=/var/lib/update +TO_VER="" +FORCE=0 +ZERO_PACKS=0 +PACK_LIST="$(mktemp pack-maker.XXXXXX)" -#NOTE: build numbers are sparse, search back for the next oldest existing build -#NOTE: in the future isn't simply -10, and when that change happens we must also -# consider which version-pair packs are meaningful for intermediate builds -# and revert builds. +usage() { + echo -e "Usage: mixer-pack-maker.sh --to VER [OPTIONS]\n" + echo -e "\t-h, --help\tDisplay this help output\n" + echo -e "\t-f, --from\tCreate delta packs using the given \"from\" version\n" + echo -e "\t-R, --repodir\tLocation of swupd-server repo to find swupd_make_pack binary\n" + echo -e "\t-S, --statedir\tThe state directory for swupd_make_pack. Defaults to /var/lib/update\n" + echo -e "\t-t, --to\tCreate packs for the given version. This option is required.\n" + echo -e "\t-x, --force\tRecreate packs if they already exist\n" + echo -e "\t-z, --zero-packs\tCreate zero packs. If only --to is specified, this is the default.\n" +} -export XZ_DEFAULTS="--threads 0" +error() { + echo -e "ERROR: $1\n" +} -VER=$1 -BACK_COUNT=$2 -UPDATEDIR=$3 -SWUPDREPO=${SWUPDREPO} +check_mom() { + local mom="$1" + if [ ! -s "$mom" ]; then + error "$mom not found" + return 1 + fi + return 0 +} -if [ -z "${BACK_COUNT}" ]; then - echo "missing to pack count" +create_pack() { + local from="$1" + local to="$2" + local bundle="$3" + + if [ -s "$STATE_DIR"/www/${to}/pack-${bundle}-from-${from}.tar ] && [ "$FORCE" -eq 0 ]; then + echo "${to}/pack-${bundle}-from-${from}.tar already exists, skipping." + else + "${REPO_DIR}"swupd_make_pack --statedir "${STATE_DIR}" ${from} ${to} ${bundle} + fi +} + +export -f create_pack + +create_packs() { + # The create_pack function uses these shell variables, so 'parallel' + # needs them exported to the environment. + export FORCE REPO_DIR STATE_DIR + cat "$PACK_LIST" | parallel --colsep '\t' create_pack {1} {2} {3} +} + +get_bundle_version() { + local mom="$1" + local bundle="$2" + + awk -v B=$bundle '$1 ~ /^M\./ && $4 == B { print $3 }' "$mom" +} + +init_delta_packs() { + local from_ver="$1" + local from_mom="$STATE_DIR/www/$from_ver/Manifest.MoM" + local to_mom="$STATE_DIR/www/$TO_VER/Manifest.MoM" + + check_mom $from_mom || return 1 + check_mom $to_mom || return 1 + + awk '$1 ~ /^M\./ { print $3, $4 }' "$from_mom" | while read FROM BUNDLE; do + local TO=$(get_bundle_version "$to_mom" "$BUNDLE") + + # If bundle does not exist in later version, skip + [ -z "$TO" ] && continue + + # If versions are equal, skip + [ "$FROM" -eq "$TO" ] && continue + + echo -e "${FROM}\t${TO}\t${BUNDLE}" >> "$PACK_LIST" + done +} + +init_zero_packs() { + local to_mom="$STATE_DIR/www/$TO_VER/Manifest.MoM" + + check_mom $to_mom || return 1 + + awk '$1 ~ /^M\./ { print $3, $4 }' "$to_mom" | while read TO BUNDLE; do + echo -e "0\t${TO}\t${BUNDLE}" >> "$PACK_LIST" + done +} + +deduplicate_pack_list() { + sort -u "$PACK_LIST" > "$PACK_LIST".new + mv "$PACK_LIST".new "$PACK_LIST" +} + +while [[ $# > 0 ]]; do + key="$1" + case $key in + -f|--from) + FROM_VERS="$FROM_VERS $2" + shift + ;; + -R|--repodir) + REPO_DIR="$2" + shift + ;; + -S|--statedir) + STATE_DIR="$2" + shift + ;; + -t|--to) + TO_VER="$2" + shift + ;; + -x|--force) + FORCE=1 + ;; + -z|--zero-packs) + ZERO_PACKS=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + error "Invalid option" + echo "" + exit 1 + ;; + esac + shift +done + +if [ -z "$TO_VER" ]; then + error "Missing required --to option" + echo "" + usage exit 1 fi -if [ -z "${VER}" ]; then - echo "missing to version" - exit 1 -fi - -if [ -z "${UPDATEDIR}" ]; then - echo "Missing STATE_DIR, using /var/lib/update/" - UPDATEDIR="/var/lib/update/" +if [ -z "$FROM_VERS" ]; then + ZERO_PACKS=1 fi # So we can call swupd_* with a sane path even if it's not / terminated -if [ ! -z "${SWUPDREPO}" ]; then - SWUPDREPO="${SWUPDREPO}/" +if [ -n "$REPO_DIR" ]; then + d=$(dirname $REPO_DIR) + b=$(basename $REPO_DIR) + REPO_DIR="${d}/${b}/" fi -SWUPDWEBDIR="${UPDATEDIR}/www" +# If delta packs are not being created, this loop does not execute +for ver in $FROM_VERS; do + # The MoMs must exist; else skip this version pair + init_delta_packs $ver || continue +done -MOM=${SWUPDWEBDIR}/${VER}/Manifest.MoM -if [ ! -e ${MOM} ]; then - echo "invalid version (no ${MOM})" - exit 1 +if [ "$ZERO_PACKS" -eq 1 ]; then + init_zero_packs fi -BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }') +deduplicate_pack_list -# build packs for all bundles changed in $VER -for BUNDLE in $BUNDLE_LIST; do - BUNDLE_VER_LIST="" - MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$VER/Manifest.MoM | grep "^previous:" | cut -f 2) - BUNDLE_VER="" - COUNT=0 - while [ $COUNT -lt $BACK_COUNT ] && [ $MANIFEST_VER -gt 0 ]; do - MOM=${SWUPDWEBDIR}/$MANIFEST_VER/Manifest.MoM - if [ -e ${MOM} ]; then - BUNDLE_VER=$(cat ${MOM} | grep " ${BUNDLE}$" | cut -f 3) - if [ "$BUNDLE_VER" != "" ]; then - BUNDLE_VER_LIST="$BUNDLE_VER $BUNDLE_VER_LIST" - MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$BUNDLE_VER/Manifest.MoM | grep "^previous:" | cut -f 2) - let COUNT=$((COUNT+1)) - else - # back in history to where bundle didn't exist yet - MANIFEST_VER=0 - fi - else - # seek backwards until a manifest if found - MANIFEST_VER=$((MANIFEST_VER-10)) - fi - done +create_packs - for v in $BUNDLE_VER_LIST; do - if [ -e ${SWUPDWEBDIR}/${VER}/pack-${BUNDLE}-from-$v.tar ]; then - echo "${VER}/pack-${BUNDLE}-from-$v.tar already exists, skipping." - else - ${SWUPDREPO}swupd_make_pack --statedir ${UPDATEDIR} $v ${VER} ${BUNDLE} & - fi - done -done +rm "$PACK_LIST" -for job in $(jobs -p) -do - wait ${job} - RET=$? - if [ "$RET" != "0" ]; then - echo "pack creation failure!" - exit $RET - fi -done # NOTE: your devops will want to expose the completed swupd server build to # trial usage at some point. This would be done via code like: # -# echo ${VER} > ${SWUPDWEBDIR}/../image/LAST_VER -# STAGING=$(cat ${SWUPDWEBDIR}/version/formatstaging/latest) -# if [ "${STAGING}" -lt "${VER}" ]; then -# echo ${VER} > ${SWUPDWEBDIR}/version/formatstaging/latest +# echo ${TO_VER} > ${STATE_DIR}/image/LAST_VER +# STAGING=$(cat ${STATE_DIR}/www/version/formatstaging/latest) +# if [ "${STAGING}" -lt "${TO_VER}" ]; then +# echo ${TO_VER} > ${STATE_DIR}/www/version/formatstaging/latest # fi