update packages file

Make sure we only parse %{version} one time while doing a catchup.

Makefile.common

@@ -166,10 +166,12 @@ preautospec-checks:
 #help to autospec.
 #help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: preautospec-checks pullrebase localreponotice clean-old-content
+	git rm --ignore-unmatch pumpAutospec || rm -f pumpAutospec
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
@@ -221,6 +223,7 @@ scanlicense:
 #help the variable BUMP_MSG is set, its value is used as the commit summary.
 #help Otherwise a generic commit summary is used.
 bump:
+	git stash
 	git pull --rebase
 	$(MAKE) bumpnogit
 	git add $(SPECFILE) release
@@ -386,6 +389,17 @@ koji-waitrepo: kojidef
 		koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
 	fi
 
+.PHONY: koji-tag
+#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
+koji-tag:
+	koji tag-pkg dist-clear $(SRPMVERS)
+
+.PHONY: koji-untag
+#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
+koji-untag:
+	koji untag-pkg dist-clear $(SRPMVERS)
+
+
 update-versions:
 
 .PHONY: update
@@ -394,17 +408,20 @@ update-versions:
 #help or update the package and push the update to koji.
 #help If no update.sh exists, update the version, autospec and push the
 #help update to koji (bumping the bump.list if found).
+update: export AUTOSPEC_UPDATE=1
 update:
+	@if [ -f update.first ]; then \
+		for pkg in $$(< update.first); do \
+			(cd "../$${pkg}" || exit 1; $(MAKE) update-versions || exit 0; $(MAKE) update) || exit; \
+		done; \
+	fi
 	@if [ -f update.sh ]; then \
 		./update.sh; \
 	else \
 		$(MAKE) -s update-versions && \
 		$(MAKE) autospec CLEANUP=1 && \
-		if [ -f bump.list ]; then \
-			$(MAKE) -s koji; \
-		else \
-			$(MAKE) -s koji-nowait; \
-		fi \
+		$(MAKE) -s koji && \
+		$(MAKE) koji-waitrepo; \
 	fi \
 
 logcheck:
@@ -428,21 +445,6 @@ repoadd: $(TOPLVL)/repo
 	$(MAKE) -s localrepocreate
 	$(MAKE) -s localrepoenable
 
-#help repostage: Stages locally-built RPMs for this package to the local RPM
-#help directory. If this directory does not yet exist, it is created and
-#help enabled.
-repostage: $(TOPLVL)/repo
-	@if [ -z "${LATEST_RPMS}" ]; then \
-		echo "No rpms found in rpms/ directory."; \
-		exit 1; \
-	fi
-	$(MAKE) -s repodel NO_CREATEREPO=1
-	@for r in ${LATEST_RPMS}; do \
-		echo "+$${rpm#rpms/}"; \
-		echo "$${rpm#rpms/}" >> .repo-index; \
-	done; \
-	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
-
 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
 repodel: $(TOPLVL)/repo
@@ -580,6 +582,78 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+#help catchup: Backport the commits from the current version to the upstream HEAD (not release).
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup:
+	$(MAKE) catchup-HEAD
+
+#help catchup-<commit|tag>: Backport the commits from the current version to the specified commit or tag.
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup-%:
+	@target=$*; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	if ! git -C results/$(PKG_NAME) rev-parse --verify --quiet "$${target}" >/dev/null; then \
+		echo "Error: Target commit/tag $${target} not found"; \
+		exit 1; \
+	fi; \
+	version=$$(rpm -q --qf '%{VERSION}\n' --specfile $(SPECFILE) | head -1); \
+	echo "Version: $${version}"; \
+	current_tag=$$(git -C results/$(PKG_NAME) tag --list | grep -E "^($(PKG_NAME)-)?v?$${version}$$") || { \
+		echo "Error: No tag found for current package version"; \
+		exit 1; \
+	}; \
+	echo "Catching up from $${current_tag} to $${target}"; \
+	for commit in $$(git -C results/$(PKG_NAME) log --reverse --pretty=oneline $${current_tag}..$${target} | cut -d' ' -f1); do \
+		$(MAKE) backport-$${commit}; \
+	done;
+
+
+#help backport-<commit>: Retrieve a commit from the upstream git repository and save it as a backport patch.
+#help The giturl is read from options.conf.
+backport-%:
+	@commit=$*; \
+	echo "Backporting commit: $${commit}"; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	full_commit=$$(git -C results/$(PKG_NAME) show --pretty=oneline $${commit} 2>/dev/null | head -1 | cut -d' ' -f 1); \
+	if [[ -z "$${full_commit}" ]]; then \
+		echo "Error: Commit for $${commit} not found"; \
+		exit 1; \
+	fi; \
+	patch=backport-$${full_commit}.patch; \
+	git -C results/$(PKG_NAME) format-patch -1 --stdout $${full_commit} > $${patch}; \
+	if [[ -s $${patch} ]]; then \
+		echo "$${patch} created"; \
+		grep -qE "^$${patch}$$" series 2>/dev/null || echo "$${patch}" >> series; \
+	else \
+		rm -f $${patch}; \
+		echo "Error: Failed to create backport patch"; \
+		exit 1; \
+	fi \
+
 .PHONY: whatrequires
 #help whatrequires: Output a list of packages that directly depend on this one,
 #help showing the subpackage-level breakdown. Each line of output has the format

Makefile.config

@@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs
 # Location where project repos are hosted.
 PRJ_BASE_URL = https://github.com/clearlinux
 
+# Location where vendor repos are hosted.
+VND_BASE_URL =
+
 # Project repos that are integral to the common tooling.
 PROJ_PKGS = autospec clr-bundles common
 

Makefile.toplevel

@@ -52,7 +52,7 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	remotepkg=$(call remotepkgname,$$pkg); \
 	[ -d packages/$$pkg ] || ( \
 	echo "Checking out: packages/$$pkg"; \
-	git clone $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
+	git clone -b main $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
 	cd packages/$$pkg; \
 	$(call gitoliteurl,packages/$$pkg); \
 	$(call subjectprefix,$$pkg); \
@@ -126,11 +126,17 @@ endif
 pull: ${PULL_PKGS}
 
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
-#help removes the associated package repos from the ./packages tree
+#help removes the associated package repos from the ./packages tree and stores
+#help it in packages_old. If it already exists in packages_old, skip it.
 clean-pkgs-dir: $(PACKAGES_FILE)
 	@for p in ${PKGS_ORPHANED}; do \
-		echo "Removing $$p from packages, it is no longer in common/packages."; \
-		rm -rf packages/$$p; \
+		mkdir -p packages_old; \
+		if [ -d "packages_old/$$p" ]; then \
+			echo "packages_old/$$p already exists, not touching it"; \
+		else \
+			echo "Moving $$p from packages to packages_old, as it is no longer in common/packages."; \
+			mv packages/$$p packages_old/$$p; \
+		fi; \
 	done
 	@echo "packages directory cleaned";
 
@@ -231,7 +237,7 @@ autospecnew: preautospecnew-checks localreponotice
 			--mock-config $(MOCK_CONFIG_VAL) \
 			--mock-opts="$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
-			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+			${NON_INTERACTIVE} ${SKIP_GIT} $(if $(CLEANUP),-C) \
 			$(URL); \
 		if [ $$? -eq 0 ]; then \
 			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
@@ -250,51 +256,6 @@ autospecnew: preautospecnew-checks localreponotice
 		exit 1; \
 	fi
 
-#help autospecnewgo: Creates a new autospec package with (dependencies)
-#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
-#help autospec will be created in the help process.
-#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
-#help to autospec.
-#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
-autospecnewgo: preautospecnew-checks localreponotice
-	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
-		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
-		exit 1; \
-	fi
-	-$(MAKE) clone_$(NAME)
-	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
-		echo "no remote repository found, creating new package repository and running autospec"; \
-		mkdir -p $(TOPLVL)/packages/$(NAME); \
-		( \
-			cd $(TOPLVL)/packages/$(NAME); \
-			git init -b main; \
-			git remote add origin $(PKG_BASE_URL)/$(NAME); \
-			$(call gitoliteurl,packages/$(NAME)); \
-			$(call subjectprefix,$(NAME)); \
-		); \
-		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
-		python3 $(TOPLVL)/projects/common/go-builder.py \
-			--target packages/$(NAME) \
-			--integrity \
-			--config "$(AUTOSPEC_CONF)" \
-			--name $(NAME) \
-			--archives $(ARCHIVES) \
-			--mock-config $(MOCK_CONFIG_VAL) \
-			--mock-opts "$(MOCK_OPTS)" \
-			$(URL); \
-		if [ $$? -eq 0 ]; then \
-			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
-			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
-			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
-			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
-		else \
-			exit 1; \
-		fi; \
-	else \
-		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
-		exit 1; \
-	fi
-
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
 repoenable: localrepoenable ;

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

go-builder.py

@@ -1,285 +0,0 @@
-#!/usr/bin/env python3
-#
-# gowrap.py - part of autospec
-# Copyright (C) 2019 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import argparse
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-
-
-def parse_args():
-    """Parse commandline arguments."""
-    parser = argparse.ArgumentParser()
-    parser.add_argument("url", help="Location of package archive")
-    parser.add_argument("-t", "--target", dest="target", action="store",
-                        default=None,
-                        help="Target location to create or reuse")
-    parser.add_argument("-c", "--config", dest="config", action="store",
-                        default="/usr/share/defaults/autospec/autospec.conf",
-                        help="Set configuration file to use")
-    parser.add_argument("-n", "--name", action="store", dest="name", default="",
-                        help="Override the package name")
-    parser.add_argument("-i", "--integrity", action="store_true",
-                        default=False,
-                        help="Search for package signature from source URL and "
-                             "attempt to verify package")
-    parser.add_argument("-m", "--mock-config", action="store", default="clear",
-                        help="Value to pass with Mock's -r option. Defaults to "
-                             "\"clear\", meaning that Mock will use "
-                             "/etc/mock/clear.cfg.")
-    parser.add_argument("-o", "--mock-opts", action="store", default="",
-                        help="Arbitrary options to pass down to mock when "
-                        "building a package.")
-    parser.add_argument('-a', "--archives", action="store",
-                        dest="archives", default=[], nargs='*',
-                        help="tarball URLs for additional source archives and"
-                        " a location for the sources to be extacted to (e.g."
-                        " http://example.com/downloads/dependency.tar.gz"
-                        " /directory/relative/to/extract/root )")
-
-    return parser.parse_args()
-
-
-def missing_dependencies(name):
-    """Check if the build failed due to missing dependencies."""
-    root_log = os.path.join("packages", name, "results", "root.log")
-    if not os.path.exists(root_log):
-        return False
-    with open(root_log, "r") as lfile:
-        for line in lfile.readlines():
-            if "No matching package to install" in line:
-                return True
-    return False
-
-
-def already_built(name, version):
-    """Check if the package and version are already built."""
-    if not os.path.exists(os.path.join("packages", name, "rpms")):
-        return False
-    vpath = os.path.join("packages", name, "versions")
-    if not os.path.exists(vpath):
-        return False
-    with open(vpath, "r") as vfile:
-            if version not in [x.strip() for x in vfile.readlines()]:
-                return False
-    return True
-
-
-def build(path, args, name, url, version):
-    """Try and build a package."""
-    if version:
-        ver_arg = ["-v", version]
-    else:
-        ver_arg = []
-    if already_built(name, version):
-        return True
-    print(f"Trying to autospec {name} - {version}")
-
-    proc = subprocess.run(["python3",
-                           f"{path}/../autospec/autospec/autospec.py", url,
-                           "-c", args.config, "-t", f"packages/{name}",
-                           "-n", name, "-m", args.mock_config,
-                           "-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
-    if proc.returncode == 0:
-        subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
-                       capture_output=True)
-    return proc.returncode == 0
-
-
-def parse_go_mod(path):
-    """Parse go.mod file for build requirements.
-
-    File content looks as follows:
-
-    module example.com/foo/bar
-
-    require (
-        github.com/BurntSushi/toml v0.3.1
-        git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
-        github.com/inconshreveable/mousetrap v1.0.0 // indirect
-        "github.com/spf13/cobra" v0.0.3
-        github.com/spf13/pflag v1.0.3 // indirect
-    )
-
-    Need to handle all require lines including //indirect.
-    Skip requires that use .git for now. May need to be handled
-    differently.
-    """
-    reqs = []
-    with open(path, "r") as gfile:
-        dep_start = False
-        for line in gfile.readlines():
-            # Ideally the mod file is generated and the format is
-            # always correct but add a few defenses just in case
-            line = line.strip()
-            if line.startswith("//"):
-                # Skip comments
-                continue
-            if dep_start:
-                # End of the require section
-                if line.startswith(")"):
-                    break
-                req = line.split()[:2]
-                req[0] = req[0].replace('"', '')
-                if req[0].endswith(".git"):
-                    continue
-                reqs.append(req)
-                continue
-            if line.startswith("require ("):
-                dep_start = True
-    return reqs
-
-
-def get_dependencies(name):
-    """Return path to the go.mod file if it exists."""
-    command = None
-    path = None
-    pdir = os.path.join("packages", name)
-    reqs = []
-    for fname in os.listdir(pdir):
-        # Find the archive
-        if fname.endswith("zip"):
-            command = ["unzip", fname]
-        elif ".tar." in fname:
-            command = ["tar", "xf", fname]
-        else:
-            command = None
-
-        if command:
-            # Decompress archive
-            tdir = tempfile.mkdtemp()
-            tfile = os.path.join(tdir, fname)
-            shutil.copyfile(os.path.join(pdir, fname), tfile)
-            proc = subprocess.run(command, cwd=tdir, capture_output=True)
-            if not proc.returncode:
-                # inspect contents for a "go.sum" file
-                for root, _, files in os.walk(tdir):
-                    if "go.mod" in files:
-                        path = f"{name}.gomod"
-                        shutil.copyfile(os.path.join(root, "go.mod"), path)
-                        reqs += parse_go_mod(path)
-                        os.unlink(path)
-            shutil.rmtree(tdir)
-
-    return reqs
-
-
-def encode_requirements(requirement):
-    """Encode the project's module and version."""
-    # Default dependency base proxy url
-    base_url = "https://proxy.golang.org/"
-    # Encoding based on
-    # https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
-    encoded_module = ""
-    for char in requirement[0]:
-        if char.isupper():
-            encoded_module += "!" + char.lower()
-        else:
-            encoded_module += char
-    encoded_version = ""
-    for char in requirement[1]:
-        if char.isupper():
-            encoded_version += "!" + char.lower()
-        else:
-            encoded_version += char
-    return (encoded_module, encoded_version)
-
-
-def initialize_package(name, url, version, path, args):
-    """Setup package and try to build it."""
-    if already_built(name, version):
-        return True
-    print(f"Trying to autospecnew {name} - {version}")
-
-    proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
-    if f"{name} already exists at" in proc.stdout.decode("utf-8"):
-        return build(path, args, name, url, version)
-    return proc.returncode == 0
-
-
-def build_recursive(path, args, name, url, version=None, success=None):
-    """Try and recursively build packages."""
-    if not success:
-        # First package being built, common utilities already run
-        # so just use autospec directly
-        success = {}
-        ret = build(path, args, name, url, version)
-    else:
-        ret = initialize_package(name, url, version, path, args)
-
-    if ret:
-        # Completed so we are done!
-        return True
-
-    if not missing_dependencies(name):
-        # Failed but not due to missing dependencies, bail
-        print(f"Unknown error building {name} - {version}")
-        return False
-
-    reqs = get_dependencies(name)
-    if not reqs:
-        # Didn't try and add anything so don't need to build again
-        print(f"Build failed with no missing requirements: {name} - {version}")
-        return False
-
-    print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
-    for req in reqs:
-        ereq = encode_requirements(req)
-        rname = "go-" + req[0].replace("/", "-")
-        rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
-        if (rname, req[1]) in success:
-            # Rebuild detected, figure out if it is okay or not
-            if success[(rname, req[1])]:
-                # Already succeeded building req short circuit success
-                continue
-            else:
-                # Encountered a build loop, bail out as this likely
-                # requires manual fixing
-                print(f"Detected build loop when building {rname} - {req[1]}")
-                return True
-        success[(rname, req[1])] = False
-        if not build_recursive(path, args, rname, rurl, req[1], success):
-            print(f"Failed to build dependency {rname} - {req[1]}")
-            return False
-
-        success[(rname, req[1])] = True
-        subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
-        subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
-
-    # Create the repo with the new packages added
-    subprocess.run(["make", "localrepocreate"], capture_output=True)
-    # Retry previously failed build assuming dependencies got added
-    print(f"Rebuilding after resolving dependencies {name} - {version}")
-    ret = build(path, args, name, url, version)
-    print("")
-
-    return ret
-
-
-def main():
-    """Start program execution."""
-    args = parse_args()
-    path = os.path.dirname(os.path.realpath(__file__))
-    return build_recursive(path, args, args.name, args.url)
-
-if __name__ == '__main__':
-    if not main():
-        sys.exit(-1)
-    print("\n\nBuild completed\n\n")

licenses-extra

@@ -5,3 +5,4 @@ GFDL-1.3+
 MIT-Opengroup
 WXwindows
 w3c
+libpng-2.0

licenses-spdx

@@ -180,6 +180,7 @@ FDK-AAC
 FSFAP
 FSFUL
 FSFULLR
+FSFULLRWD
 FTL
 Fair
 Frameworx-1.0
@@ -252,6 +253,7 @@ JPNIC
 JSON
 Jam
 JasPer-2.0
+Knuth-CTAN
 LAL-1.2
 LAL-1.3
 LGPL-2.0
@@ -471,6 +473,7 @@ Zlib
 blessing
 bzip2-1.0.5
 bzip2-1.0.6
+checkmk
 copyleft-next-0.3.0
 copyleft-next-0.3.1
 curl
@@ -485,6 +488,7 @@ iMatix
 libpng-2.0
 libselinux-1.0
 libtiff
+libutil-David-Nugent
 mpi-permissive
 mpich2
 mplus

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:

vendor.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+import requests
+from git import Repo
+
+
+def vendor_check():
+    if not os.path.isfile('options.conf'):
+        return False
+    config = configparser.ConfigParser(interpolation=None)
+    config.read('options.conf')
+    if 'autospec' not in config.sections():
+        return False
+    if vendor := config['autospec'].get('cargo_vendor'):
+        if vendor == "true":
+            return 'cargo'
+
+    return False
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
+    parser.add_argument('url')
+    parser.add_argument('name')
+    parser.add_argument('git')
+    return parser.parse_args()
+
+
+def setup_content(url):
+    tdir = tempfile.mkdtemp()
+    outfile = os.path.join(tdir, os.path.basename(url))
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()
+    with open(outfile, 'wb') as cfile:
+        cfile.write(response.content)
+
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
+    os.remove(outfile)
+    return tdir
+
+
+def setup_cargo_vendor(path):
+    cargo_paths = []
+    for dirpath, _, files in os.walk(path):
+        for fname in files:
+            if fname == "Cargo.toml":
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
+
+
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
+    git_uri = os.path.join(git, name)
+    vendor_path = os.path.join(tmpdir, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    vendor_git = os.path.join(vendor_path, '.git')
+    if not os.path.isdir(vendor_git):
+        # initialize a git repo
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
+        subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    shutil.rmtree(vendor_path)
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
+                                   check=True, stdout=subprocess.PIPE,
+                                   universal_newlines=True).stdout
+    with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
+        gafile.write("* text=false\n")
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    repo = Repo(vendor_path)
+    if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
+        # Always use the newest tag as sometimes a new tag will
+        # be created but the package won't be updated to use it
+        # for a different failure reason.
+        tag = sorted(repo.tags, key=lambda x: x.name, reverse=True)[0]
+        return tag, cargo_vendors
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    gmt = time.gmtime()
+    tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
+    subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
+                   check=True, stdout=subprocess.DEVNULL)
+    subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    time.sleep(30)
+    return tag, cargo_vendors
+
+
+def update_cargo_sources(name, tag, cargo_vendors):
+    makefile = []
+    options = []
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
+                                 'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
+    with open('Makefile', encoding='utf8') as mfile:
+        for line in mfile.readlines():
+            if line.startswith('ARCHIVES'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
+                makefile.append(new_archives)
+            else:
+                makefile.append(line)
+    with open('Makefile', 'w', encoding='utf8') as mfile:
+        mfile.writelines(makefile)
+
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+    with open('cargo_vendors', 'w', encoding='utf8') as cfile:
+        cfile.write(cargo_vendors)
+
+
+def main():
+    args = get_args()
+
+    vtype = vendor_check()
+    if not vtype:
+        print(args.archives)
+        return
+
+    tdir = setup_content(args.url)
+    if vtype == 'cargo':
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
+            print(args.archives)
+        else:
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
+            update_cargo_sources(args.name, tag, cargo_vendors)
+    shutil.rmtree(tdir)
+
+
+if __name__ == '__main__':
+    main()