From 4da94d56f12ecdbe5ef78d59b06af3161fbdb2ce Mon Sep 17 00:00:00 2001 From: Patrick McCarty Date: Mon, 29 Aug 2016 16:56:19 -0700 Subject: [PATCH] pack maker rewrite: support zero packs, change delta pack creation logic The first aim of this patch is to centralize pack creation for SWUPD, adding the ability to create zero packs in addition to delta packs. Secondly, to make delta pack creation more flexible, instead of considering the last N times a bundle changed, require the user to specify the "from" version delta packs should be based on. More than one --from option can be passed if several sets of delta packs need to be built. To facilitate these different modes of operation, I added option parsing, also with --help output and some examples documented in the script header. Additionally, GNU Parallel is used to parallelize pack creation, instead of shell job control facilities. Signed-off-by: Patrick McCarty --- pack-maker.sh | 266 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 85 deletions(-) diff --git a/pack-maker.sh b/pack-maker.sh index d325dd4..3546cb0 100755 --- a/pack-maker.sh +++ b/pack-maker.sh @@ -1,110 +1,206 @@ -#!/bin/bash -e -# usage: -# mixer-pack-maker.sh -# mixer-pack-maker.sh 2820 2 -# mixer-pack-maker.sh 2730 3 +#!/bin/bash +# +# Usage: +# mixer-pack-maker.sh --to VER [OPTIONS] +# +# Create zero packs and/or delta packs for a given swupd build VER. +# +# The presence of the --from option enables delta pack creation from the given +# version. More than one --from option may be specified. If both zero and delta +# packs should be created at the same time, pass the --zero option in addition +# to the --from option. +# +# To force recreation of all packs (zero and/or delta), use the --force option. +# +# Example invocations: +# +# 1) Create zero packs for build 7000 +# +# mixer-pack-maker.sh --to 7000 +# +# 2) Create delta packs from build 6000 to 7000, and from 6500 to 7000 +# +# mixer-pack-maker.sh --to 7000 --from 6000 --from 6500 +# +# 3) Create delta packs from build 6900 to 7000, and zero packs for build 7000 +# +# mixer-pack-maker.sh --to 7000 --from 6900 --zero # -# For each bundle changed as of the Manifest.MoM at the , -# create version-pair packs going back -# versions of that bundle. For example if bundle "editors" changed in 2650 -# and the script is run as "pack_maker.sh 2650 2", it will find that -# "editors" last two change points were 2550 and 2260 and thus create a -# 2550-to-2650 and a 2260-to-2650 version pair pack for the "editors" -# bundle. -#NOTE: fullfiles must be created before calling this script -#NOTE: zero packs need similarly created +FROM_VERS="" +REPO_DIR="" +STATE_DIR=/var/lib/update +TO_VER="" +FORCE=0 +ZERO_PACKS=0 +PACK_LIST="$(mktemp pack-maker.XXXXXX)" -#NOTE: build numbers are sparse, search back for the next oldest existing build -#NOTE: in the future isn't simply -10, and when that change happens we must also -# consider which version-pair packs are meaningful for intermediate builds -# and revert builds. +usage() { + echo -e "Usage: mixer-pack-maker.sh --to VER [OPTIONS]\n" + echo -e "\t-h, --help\tDisplay this help output\n" + echo -e "\t-f, --from\tCreate delta packs using the given \"from\" version\n" + echo -e "\t-R, --repodir\tLocation of swupd-server repo to find swupd_make_pack binary\n" + echo -e "\t-S, --statedir\tThe state directory for swupd_make_pack. Defaults to /var/lib/update\n" + echo -e "\t-t, --to\tCreate packs for the given version. This option is required.\n" + echo -e "\t-x, --force\tRecreate packs if they already exist\n" + echo -e "\t-z, --zero-packs\tCreate zero packs. If only --to is specified, this is the default.\n" +} -export XZ_DEFAULTS="--threads 0" +error() { + echo -e "ERROR: $1\n" +} -VER=$1 -BACK_COUNT=$2 -UPDATEDIR=$3 -SWUPDREPO=${SWUPDREPO} +check_mom() { + local mom="$1" + if [ ! -s "$mom" ]; then + error "$mom not found" + return 1 + fi + return 0 +} -if [ -z "${BACK_COUNT}" ]; then - echo "missing to pack count" +create_pack() { + local from="$1" + local to="$2" + local bundle="$3" + + if [ -s "$STATE_DIR"/www/${to}/pack-${bundle}-from-${from}.tar ] && [ "$FORCE" -eq 0 ]; then + echo "${to}/pack-${bundle}-from-${from}.tar already exists, skipping." + else + "${REPO_DIR}"swupd_make_pack --statedir "${STATE_DIR}" ${from} ${to} ${bundle} + fi +} + +export -f create_pack + +create_packs() { + # The create_pack function uses these shell variables, so 'parallel' + # needs them exported to the environment. + export FORCE REPO_DIR STATE_DIR + cat "$PACK_LIST" | parallel --colsep '\t' create_pack {1} {2} {3} +} + +get_bundle_version() { + local mom="$1" + local bundle="$2" + + awk -v B=$bundle '$1 ~ /^M\./ && $4 == B { print $3 }' "$mom" +} + +init_delta_packs() { + local from_ver="$1" + local from_mom="$STATE_DIR/www/$from_ver/Manifest.MoM" + local to_mom="$STATE_DIR/www/$TO_VER/Manifest.MoM" + + check_mom $from_mom || return 1 + check_mom $to_mom || return 1 + + awk '$1 ~ /^M\./ { print $3, $4 }' "$from_mom" | while read FROM BUNDLE; do + local TO=$(get_bundle_version "$to_mom" "$BUNDLE") + + # If bundle does not exist in later version, skip + [ -z "$TO" ] && continue + + # If versions are equal, skip + [ "$FROM" -eq "$TO" ] && continue + + echo -e "${FROM}\t${TO}\t${BUNDLE}" >> "$PACK_LIST" + done +} + +init_zero_packs() { + local to_mom="$STATE_DIR/www/$TO_VER/Manifest.MoM" + + check_mom $to_mom || return 1 + + awk '$1 ~ /^M\./ { print $3, $4 }' "$to_mom" | while read TO BUNDLE; do + echo -e "0\t${TO}\t${BUNDLE}" >> "$PACK_LIST" + done +} + +deduplicate_pack_list() { + sort -u "$PACK_LIST" > "$PACK_LIST".new + mv "$PACK_LIST".new "$PACK_LIST" +} + +while [[ $# > 0 ]]; do + key="$1" + case $key in + -f|--from) + FROM_VERS="$FROM_VERS $2" + shift + ;; + -R|--repodir) + REPO_DIR="$2" + shift + ;; + -S|--statedir) + STATE_DIR="$2" + shift + ;; + -t|--to) + TO_VER="$2" + shift + ;; + -x|--force) + FORCE=1 + ;; + -z|--zero-packs) + ZERO_PACKS=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + error "Invalid option" + echo "" + exit 1 + ;; + esac + shift +done + +if [ -z "$TO_VER" ]; then + error "Missing required --to option" + echo "" + usage exit 1 fi -if [ -z "${VER}" ]; then - echo "missing to version" - exit 1 -fi - -if [ -z "${UPDATEDIR}" ]; then - echo "Missing STATE_DIR, using /var/lib/update/" - UPDATEDIR="/var/lib/update/" +if [ -z "$FROM_VERS" ]; then + ZERO_PACKS=1 fi # So we can call swupd_* with a sane path even if it's not / terminated -if [ ! -z "${SWUPDREPO}" ]; then - SWUPDREPO="${SWUPDREPO}/" +if [ -n "$REPO_DIR" ]; then + d=$(dirname $REPO_DIR) + b=$(basename $REPO_DIR) + REPO_DIR="${d}/${b}/" fi -SWUPDWEBDIR="${UPDATEDIR}/www" +# If delta packs are not being created, this loop does not execute +for ver in $FROM_VERS; do + # The MoMs must exist; else skip this version pair + init_delta_packs $ver || continue +done -MOM=${SWUPDWEBDIR}/${VER}/Manifest.MoM -if [ ! -e ${MOM} ]; then - echo "invalid version (no ${MOM})" - exit 1 +if [ "$ZERO_PACKS" -eq 1 ]; then + init_zero_packs fi -BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }') +deduplicate_pack_list -# build packs for all bundles changed in $VER -for BUNDLE in $BUNDLE_LIST; do - BUNDLE_VER_LIST="" - MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$VER/Manifest.MoM | grep "^previous:" | cut -f 2) - BUNDLE_VER="" - COUNT=0 - while [ $COUNT -lt $BACK_COUNT ] && [ $MANIFEST_VER -gt 0 ]; do - MOM=${SWUPDWEBDIR}/$MANIFEST_VER/Manifest.MoM - if [ -e ${MOM} ]; then - BUNDLE_VER=$(cat ${MOM} | grep " ${BUNDLE}$" | cut -f 3) - if [ "$BUNDLE_VER" != "" ]; then - BUNDLE_VER_LIST="$BUNDLE_VER $BUNDLE_VER_LIST" - MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$BUNDLE_VER/Manifest.MoM | grep "^previous:" | cut -f 2) - let COUNT=$((COUNT+1)) - else - # back in history to where bundle didn't exist yet - MANIFEST_VER=0 - fi - else - # seek backwards until a manifest if found - MANIFEST_VER=$((MANIFEST_VER-10)) - fi - done +create_packs - for v in $BUNDLE_VER_LIST; do - if [ -e ${SWUPDWEBDIR}/${VER}/pack-${BUNDLE}-from-$v.tar ]; then - echo "${VER}/pack-${BUNDLE}-from-$v.tar already exists, skipping." - else - ${SWUPDREPO}swupd_make_pack --statedir ${UPDATEDIR} $v ${VER} ${BUNDLE} & - fi - done -done +rm "$PACK_LIST" -for job in $(jobs -p) -do - wait ${job} - RET=$? - if [ "$RET" != "0" ]; then - echo "pack creation failure!" - exit $RET - fi -done # NOTE: your devops will want to expose the completed swupd server build to # trial usage at some point. This would be done via code like: # -# echo ${VER} > ${SWUPDWEBDIR}/../image/LAST_VER -# STAGING=$(cat ${SWUPDWEBDIR}/version/formatstaging/latest) -# if [ "${STAGING}" -lt "${VER}" ]; then -# echo ${VER} > ${SWUPDWEBDIR}/version/formatstaging/latest +# echo ${TO_VER} > ${STATE_DIR}/image/LAST_VER +# STAGING=$(cat ${STATE_DIR}/www/version/formatstaging/latest) +# if [ "${STAGING}" -lt "${TO_VER}" ]; then +# echo ${TO_VER} > ${STATE_DIR}/www/version/formatstaging/latest # fi