From cdc2c8a9894f1a5afe616c0ee504dacc4ad0be4f Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 12 Jan 2023 16:48:48 -0800 Subject: [PATCH] Add support for "external pins" in "diff-pr.sh" This uses `crane` to download some of the JSON files of the remote images to give us at least *some* amount of diffing between updates to external pins. --- .github/workflows/munge-pr.yml | 32 ++++++++++++++----- Dockerfile | 7 +++++ diff-pr.sh | 57 +++++++++++++++++++++++++--------- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/.github/workflows/munge-pr.yml b/.github/workflows/munge-pr.yml index b59409c0f..d4aab826f 100644 --- a/.github/workflows/munge-pr.yml +++ b/.github/workflows/munge-pr.yml @@ -31,6 +31,15 @@ jobs: run: | (set +x; echo "::stop-commands::$(echo -n ${{ github.token }} | sha256sum | head -c 64)") git fetch --quiet https://github.com/docker-library/official-images.git master + externalPins="$(git diff --no-renames --name-only FETCH_HEAD...HEAD -- '.external-pins/*/**')" + externalPinTags="$( + if [ -n "$externalPins" ]; then + # doing backflips to run "tag.sh" from master instead of from the PR + git show FETCH_HEAD:.external-pins/tag.sh > ~/master-external-pins-tag.sh + chmod +x ~/master-external-pins-tag.sh + ~/master-external-pins-tag.sh $externalPins + fi + )" images="$(git diff --no-renames --name-only FETCH_HEAD...HEAD -- library/)" if [ -n "$images" ]; then new="$(git diff --no-renames --name-only --diff-filter=A FETCH_HEAD...HEAD -- $images)" @@ -39,22 +48,28 @@ jobs: new= deleted= fi - export images new deleted + export images new deleted externalPins externalPinTags images="$(jq -cn ' (env.images | rtrimstr("\n") | split("\n")) as $images | (env.new | rtrimstr("\n") | split("\n")) as $new | (env.deleted | rtrimstr("\n") | split("\n")) as $deleted + | (env.externalPins | rtrimstr("\n") | split("\n")) as $externalPins + | (env.externalPinTags | rtrimstr("\n") | split("\n")) as $externalPinTags | { images: $images, count: ($images | length), new: $new, deleted: $deleted, + externalPins: $externalPins, + externalPinTags: $externalPinTags, + externalPinsCount: ($externalPins | length), } + | .imagesAndExternalPinsCount = (.count + .externalPinsCount) # man, I *really* do not love GitHub Actions expressions... ')" jq . <<<"$images" set +x echo "::$(echo -n ${{ github.token }} | sha256sum | head -c 64)::" - echo "::set-output name=images::$images" + echo "images=$images" >> "$GITHUB_OUTPUT" outputs: images: '${{ steps.gather.outputs.images }}' @@ -62,17 +77,20 @@ jobs: name: Apply Labels runs-on: ubuntu-latest needs: gather - if: fromJSON(needs.gather.outputs.images).count > 0 + if: fromJSON(needs.gather.outputs.images).imagesAndExternalPinsCount > 0 steps: - name: Apply Labels - uses: actions/github-script@v5 + uses: actions/github-script@v6 env: IMAGES: ${{ needs.gather.outputs.images }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const data = JSON.parse(process.env.IMAGES); - var labels = data.images; + var labels = [ + ...data.images, // "library/debian", ... + ...new Set(data.externalPinTags.map(x => 'external/' + x.replace(/:.+$/, ''))), // "external/mcr.microsoft.com/windows/servercore", ... + ]; if (data.new.length > 0) { labels.push('new-image'); } @@ -88,7 +106,7 @@ jobs: name: Diff Comment runs-on: ubuntu-latest needs: gather - if: fromJSON(needs.gather.outputs.images).count > 0 + if: fromJSON(needs.gather.outputs.images).imagesAndExternalPinsCount > 0 steps: - uses: actions/checkout@v3 with: @@ -120,7 +138,7 @@ jobs: run: | docker run --rm --read-only --tmpfs /tmp oisupport/bashbrew:diff-pr ./diff-pr.sh "$GITHUB_PR_NUMBER" | tee "$GITHUB_WORKSPACE/oi-pr.diff" - name: Comment - uses: actions/github-script@v5 + uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/Dockerfile b/Dockerfile index 44f0ad5fc..d6d21662e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,18 @@ RUN set -eux; \ gawk \ # tar -tf in diff-pr.sh bzip2 \ +# jq for diff-pr.sh + jq \ ; \ rm -rf /var/lib/apt/lists/* ENV DIR /usr/src/official-images ENV BASHBREW_LIBRARY $DIR/library +# crane for diff-pr.sh +# https://gcr.io/go-containerregistry/crane:latest +# https://explore.ggcr.dev/?image=gcr.io/go-containerregistry/crane:latest +COPY --from=gcr.io/go-containerregistry/crane@sha256:d0e5cc313e7388a573bb4cfb980a935bb740c5787df7d90f7066b8e8146455ed /ko-app/crane /usr/local/bin/ + WORKDIR $DIR COPY . $DIR diff --git a/diff-pr.sh b/diff-pr.sh index dc78b0e7c..eb2fcff0c 100755 --- a/diff-pr.sh +++ b/diff-pr.sh @@ -79,10 +79,15 @@ else "$diffDir" HEAD:refs/heads/pull fi +externalPins= if [ "$#" -eq 0 ]; then + externalPins="$(git -C oi/.external-pins diff --no-renames --name-only HEAD...pull -- '*/**')" + images="$(git -C oi/library diff --no-renames --name-only HEAD...pull -- .)" - [ -n "$images" ] || exit 0 - images="$(xargs -n1 basename <<<"$images")" + if [ -z "$images" ] && [ -z "$externalPins" ]; then + exit 0 + fi + images="$(xargs -rn1 basename <<<"$images")" set -- $images fi @@ -261,21 +266,45 @@ templateLastTags=' ' _metadata-files() { - bashbrew list "$@" 2>>temp/_bashbrew.err | sort -uV > temp/_bashbrew-list || : + if [ "$#" -gt 0 ]; then + bashbrew list "$@" 2>>temp/_bashbrew.err | sort -uV > temp/_bashbrew-list || : - "$diffDir/_bashbrew-cat-sorted.sh" "$@" 2>>temp/_bashbrew.err > temp/_bashbrew-cat || : + "$diffDir/_bashbrew-cat-sorted.sh" "$@" 2>>temp/_bashbrew.err > temp/_bashbrew-cat || : - bashbrew list --uniq "$@" \ - | sort -V \ - | xargs -r bashbrew list --uniq --build-order 2>>temp/_bashbrew.err \ - | xargs -r bashbrew cat --format "$templateLastTags" 2>>temp/_bashbrew.err \ - > temp/_bashbrew-list-build-order || : + bashbrew list --uniq "$@" \ + | sort -V \ + | xargs -r bashbrew list --uniq --build-order 2>>temp/_bashbrew.err \ + | xargs -r bashbrew cat --format "$templateLastTags" 2>>temp/_bashbrew.err \ + > temp/_bashbrew-list-build-order || : - script="$(bashbrew cat --format "$template" "$@")" - mkdir tar - ( eval "$script" | tar -xiC tar ) - copy-tar tar temp - rm -rf tar + script="$(bashbrew cat --format "$template" "$@")" + mkdir tar + ( eval "$script" | tar -xiC tar ) + copy-tar tar temp + rm -rf tar + fi + + if [ -n "$externalPins" ] && command -v crane &> /dev/null && command -v jq &> /dev/null; then + local file + for file in $externalPins; do + [ -e "oi/$file" ] || continue + local pin digest dir + pin="$("$diffDir/.external-pins/tag.sh" "$file")" + digest="$(< "oi/$file")" + dir="temp/$file" + mkdir -p "$dir" + bashbrew remote arches --json "$pin@$digest" | jq -S . > "$dir/bashbrew.json" + local manifests manifest + manifests="$(jq -r '[ .arches[][].digest | @sh ] | join(" ")' "$dir/bashbrew.json")" + eval "manifests=( $manifests )" + for manifest in "${manifests[@]}"; do + crane manifest "$pin@$manifest" | jq -S . > "$dir/manifest-${manifest//:/_}.json" + local config + config="$(jq -r '.config.digest' "$dir/manifest-${manifest//:/_}.json")" + crane blob "$pin@$config" | jq -S . > "$dir/manifest-${manifest//:/_}-config.json" + done + done + fi } mkdir temp