archive repository

We support tag names like
${package}-${version}
${package}-v${version}
${version}
v${version}

Now add support for things like
${package}-foo-bar-${version}
foo-bar-${version}

for example:
vulkan-sdk-1.3.275.0 for SPIRV-Headers

Makefile.common

@@ -157,10 +157,6 @@ pullrebase:
 		fi \
 	fi
 
-# help vendor: Attempt to create an updated vendor tar archive if needed
-vendor: preautospec-checks pullrebase clean-old-content
-	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
-
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -170,12 +166,13 @@ preautospec-checks:
 #help to autospec.
 #help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: preautospec-checks pullrebase localreponotice clean-old-content
+	git rm --ignore-unmatch pumpAutospec || rm -f pumpAutospec
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
-	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \
@@ -226,6 +223,7 @@ scanlicense:
 #help the variable BUMP_MSG is set, its value is used as the commit summary.
 #help Otherwise a generic commit summary is used.
 bump:
+	git stash
 	git pull --rebase
 	$(MAKE) bumpnogit
 	git add $(SPECFILE) release
@@ -391,6 +389,17 @@ koji-waitrepo: kojidef
 		koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
 	fi
 
+.PHONY: koji-tag
+#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
+koji-tag:
+	koji tag-pkg dist-clear $(SRPMVERS)
+
+.PHONY: koji-untag
+#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
+koji-untag:
+	koji untag-pkg dist-clear $(SRPMVERS)
+
+
 update-versions:
 
 .PHONY: update
@@ -399,17 +408,20 @@ update-versions:
 #help or update the package and push the update to koji.
 #help If no update.sh exists, update the version, autospec and push the
 #help update to koji (bumping the bump.list if found).
+update: export AUTOSPEC_UPDATE=1
 update:
+	@if [ -f update.first ]; then \
+		for pkg in $$(< update.first); do \
+			(cd "../$${pkg}" || exit 1; $(MAKE) update-versions || exit 0; $(MAKE) update) || exit; \
+		done; \
+	fi
 	@if [ -f update.sh ]; then \
 		./update.sh; \
 	else \
 		$(MAKE) -s update-versions && \
 		$(MAKE) autospec CLEANUP=1 && \
-		if [ -f bump.list ]; then \
-			$(MAKE) -s koji; \
-		else \
-			$(MAKE) -s koji-nowait; \
-		fi \
+		$(MAKE) -s koji && \
+		$(MAKE) koji-waitrepo; \
 	fi \
 
 logcheck:
@@ -433,21 +445,6 @@ repoadd: $(TOPLVL)/repo
 	$(MAKE) -s localrepocreate
 	$(MAKE) -s localrepoenable
 
-#help repostage: Stages locally-built RPMs for this package to the local RPM
-#help directory. If this directory does not yet exist, it is created and
-#help enabled.
-repostage: $(TOPLVL)/repo
-	@if [ -z "${LATEST_RPMS}" ]; then \
-		echo "No rpms found in rpms/ directory."; \
-		exit 1; \
-	fi
-	$(MAKE) -s repodel NO_CREATEREPO=1
-	@for r in ${LATEST_RPMS}; do \
-		echo "+$${rpm#rpms/}"; \
-		echo "$${rpm#rpms/}" >> .repo-index; \
-	done; \
-	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
-
 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
 repodel: $(TOPLVL)/repo
@@ -585,6 +582,78 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+#help catchup: Backport the commits from the current version to the upstream HEAD (not release).
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup:
+	$(MAKE) catchup-HEAD
+
+#help catchup-<commit|tag>: Backport the commits from the current version to the specified commit or tag.
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup-%:
+	@target=$*; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	if ! git -C results/$(PKG_NAME) rev-parse --verify --quiet "$${target}" >/dev/null; then \
+		echo "Error: Target commit/tag $${target} not found"; \
+		exit 1; \
+	fi; \
+	version=$$(rpm -q --qf '%{VERSION}\n' --specfile $(SPECFILE) | head -1); \
+	echo "Version: $${version}"; \
+	current_tag=$$(git -C results/$(PKG_NAME) tag --list | grep -E "^($(PKG_NAME)-)?(.+-|v)?$${version}$$") || { \
+		echo "Error: No tag found for current package version"; \
+		exit 1; \
+	}; \
+	echo "Catching up from $${current_tag} to $${target}"; \
+	for commit in $$(git -C results/$(PKG_NAME) log --reverse --pretty=oneline $${current_tag}..$${target} | cut -d' ' -f1); do \
+		$(MAKE) backport-$${commit}; \
+	done;
+
+
+#help backport-<commit>: Retrieve a commit from the upstream git repository and save it as a backport patch.
+#help The giturl is read from options.conf.
+backport-%:
+	@commit=$*; \
+	echo "Backporting commit: $${commit}"; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	full_commit=$$(git -C results/$(PKG_NAME) show --pretty=oneline $${commit} 2>/dev/null | head -1 | cut -d' ' -f 1); \
+	if [[ -z "$${full_commit}" ]]; then \
+		echo "Error: Commit for $${commit} not found"; \
+		exit 1; \
+	fi; \
+	patch=backport-$${full_commit}.patch; \
+	git -C results/$(PKG_NAME) format-patch -1 --stdout $${full_commit} > $${patch}; \
+	if [[ -s $${patch} ]]; then \
+		echo "$${patch} created"; \
+		grep -qE "^$${patch}$$" series 2>/dev/null || echo "$${patch}" >> series; \
+	else \
+		rm -f $${patch}; \
+		echo "Error: Failed to create backport patch"; \
+		exit 1; \
+	fi \
+
 .PHONY: whatrequires
 #help whatrequires: Output a list of packages that directly depend on this one,
 #help showing the subpackage-level breakdown. Each line of output has the format

Makefile.toplevel

@@ -126,11 +126,17 @@ endif
 pull: ${PULL_PKGS}
 
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
-#help removes the associated package repos from the ./packages tree
+#help removes the associated package repos from the ./packages tree and stores
+#help it in packages_old. If it already exists in packages_old, skip it.
 clean-pkgs-dir: $(PACKAGES_FILE)
 	@for p in ${PKGS_ORPHANED}; do \
-		echo "Removing $$p from packages, it is no longer in common/packages."; \
-		rm -rf packages/$$p; \
+		mkdir -p packages_old; \
+		if [ -d "packages_old/$$p" ]; then \
+			echo "packages_old/$$p already exists, not touching it"; \
+		else \
+			echo "Moving $$p from packages to packages_old, as it is no longer in common/packages."; \
+			mv packages/$$p packages_old/$$p; \
+		fi; \
 	done
 	@echo "packages directory cleaned";
 
@@ -231,7 +237,7 @@ autospecnew: preautospecnew-checks localreponotice
 			--mock-config $(MOCK_CONFIG_VAL) \
 			--mock-opts="$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
-			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+			${NON_INTERACTIVE} ${SKIP_GIT} $(if $(CLEANUP),-C) \
 			$(URL); \
 		if [ $$? -eq 0 ]; then \
 			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
@@ -250,51 +256,6 @@ autospecnew: preautospecnew-checks localreponotice
 		exit 1; \
 	fi
 
-#help autospecnewgo: Creates a new autospec package with (dependencies)
-#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
-#help autospec will be created in the help process.
-#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
-#help to autospec.
-#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
-autospecnewgo: preautospecnew-checks localreponotice
-	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
-		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
-		exit 1; \
-	fi
-	-$(MAKE) clone_$(NAME)
-	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
-		echo "no remote repository found, creating new package repository and running autospec"; \
-		mkdir -p $(TOPLVL)/packages/$(NAME); \
-		( \
-			cd $(TOPLVL)/packages/$(NAME); \
-			git init -b main; \
-			git remote add origin $(PKG_BASE_URL)/$(NAME); \
-			$(call gitoliteurl,packages/$(NAME)); \
-			$(call subjectprefix,$(NAME)); \
-		); \
-		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
-		python3 $(TOPLVL)/projects/common/go-builder.py \
-			--target packages/$(NAME) \
-			--integrity \
-			--config "$(AUTOSPEC_CONF)" \
-			--name $(NAME) \
-			--archives $(ARCHIVES) \
-			--mock-config $(MOCK_CONFIG_VAL) \
-			--mock-opts "$(MOCK_OPTS)" \
-			$(URL); \
-		if [ $$? -eq 0 ]; then \
-			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
-			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
-			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
-			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
-		else \
-			exit 1; \
-		fi; \
-	else \
-		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
-		exit 1; \
-	fi
-
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
 repoenable: localrepoenable ;

README.md

@@ -1,3 +1,10 @@
+## DISCONTINUATION OF PROJECT. 
+
+This project will no longer be maintained by Intel. 
+
+Intel will not provide or guarantee development of or support for this project, including but not limited to, maintenance, bug fixes, new releases or updates. Patches to this project are no longer accepted by Intel. If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the community, please create your own fork of the project.
+
+Contact: webadmin@linux.intel.com  
 # Developer tooling framework for Clear Linux
 
 This repository includes scripts, configuration files, and makefiles that

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

go-builder.py

@@ -1,285 +0,0 @@
-#!/usr/bin/env python3
-#
-# gowrap.py - part of autospec
-# Copyright (C) 2019 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import argparse
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-
-
-def parse_args():
-    """Parse commandline arguments."""
-    parser = argparse.ArgumentParser()
-    parser.add_argument("url", help="Location of package archive")
-    parser.add_argument("-t", "--target", dest="target", action="store",
-                        default=None,
-                        help="Target location to create or reuse")
-    parser.add_argument("-c", "--config", dest="config", action="store",
-                        default="/usr/share/defaults/autospec/autospec.conf",
-                        help="Set configuration file to use")
-    parser.add_argument("-n", "--name", action="store", dest="name", default="",
-                        help="Override the package name")
-    parser.add_argument("-i", "--integrity", action="store_true",
-                        default=False,
-                        help="Search for package signature from source URL and "
-                             "attempt to verify package")
-    parser.add_argument("-m", "--mock-config", action="store", default="clear",
-                        help="Value to pass with Mock's -r option. Defaults to "
-                             "\"clear\", meaning that Mock will use "
-                             "/etc/mock/clear.cfg.")
-    parser.add_argument("-o", "--mock-opts", action="store", default="",
-                        help="Arbitrary options to pass down to mock when "
-                        "building a package.")
-    parser.add_argument('-a', "--archives", action="store",
-                        dest="archives", default=[], nargs='*',
-                        help="tarball URLs for additional source archives and"
-                        " a location for the sources to be extacted to (e.g."
-                        " http://example.com/downloads/dependency.tar.gz"
-                        " /directory/relative/to/extract/root )")
-
-    return parser.parse_args()
-
-
-def missing_dependencies(name):
-    """Check if the build failed due to missing dependencies."""
-    root_log = os.path.join("packages", name, "results", "root.log")
-    if not os.path.exists(root_log):
-        return False
-    with open(root_log, "r") as lfile:
-        for line in lfile.readlines():
-            if "No matching package to install" in line:
-                return True
-    return False
-
-
-def already_built(name, version):
-    """Check if the package and version are already built."""
-    if not os.path.exists(os.path.join("packages", name, "rpms")):
-        return False
-    vpath = os.path.join("packages", name, "versions")
-    if not os.path.exists(vpath):
-        return False
-    with open(vpath, "r") as vfile:
-            if version not in [x.strip() for x in vfile.readlines()]:
-                return False
-    return True
-
-
-def build(path, args, name, url, version):
-    """Try and build a package."""
-    if version:
-        ver_arg = ["-v", version]
-    else:
-        ver_arg = []
-    if already_built(name, version):
-        return True
-    print(f"Trying to autospec {name} - {version}")
-
-    proc = subprocess.run(["python3",
-                           f"{path}/../autospec/autospec/autospec.py", url,
-                           "-c", args.config, "-t", f"packages/{name}",
-                           "-n", name, "-m", args.mock_config,
-                           "-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
-    if proc.returncode == 0:
-        subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
-                       capture_output=True)
-    return proc.returncode == 0
-
-
-def parse_go_mod(path):
-    """Parse go.mod file for build requirements.
-
-    File content looks as follows:
-
-    module example.com/foo/bar
-
-    require (
-        github.com/BurntSushi/toml v0.3.1
-        git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
-        github.com/inconshreveable/mousetrap v1.0.0 // indirect
-        "github.com/spf13/cobra" v0.0.3
-        github.com/spf13/pflag v1.0.3 // indirect
-    )
-
-    Need to handle all require lines including //indirect.
-    Skip requires that use .git for now. May need to be handled
-    differently.
-    """
-    reqs = []
-    with open(path, "r") as gfile:
-        dep_start = False
-        for line in gfile.readlines():
-            # Ideally the mod file is generated and the format is
-            # always correct but add a few defenses just in case
-            line = line.strip()
-            if line.startswith("//"):
-                # Skip comments
-                continue
-            if dep_start:
-                # End of the require section
-                if line.startswith(")"):
-                    break
-                req = line.split()[:2]
-                req[0] = req[0].replace('"', '')
-                if req[0].endswith(".git"):
-                    continue
-                reqs.append(req)
-                continue
-            if line.startswith("require ("):
-                dep_start = True
-    return reqs
-
-
-def get_dependencies(name):
-    """Return path to the go.mod file if it exists."""
-    command = None
-    path = None
-    pdir = os.path.join("packages", name)
-    reqs = []
-    for fname in os.listdir(pdir):
-        # Find the archive
-        if fname.endswith("zip"):
-            command = ["unzip", fname]
-        elif ".tar." in fname:
-            command = ["tar", "xf", fname]
-        else:
-            command = None
-
-        if command:
-            # Decompress archive
-            tdir = tempfile.mkdtemp()
-            tfile = os.path.join(tdir, fname)
-            shutil.copyfile(os.path.join(pdir, fname), tfile)
-            proc = subprocess.run(command, cwd=tdir, capture_output=True)
-            if not proc.returncode:
-                # inspect contents for a "go.sum" file
-                for root, _, files in os.walk(tdir):
-                    if "go.mod" in files:
-                        path = f"{name}.gomod"
-                        shutil.copyfile(os.path.join(root, "go.mod"), path)
-                        reqs += parse_go_mod(path)
-                        os.unlink(path)
-            shutil.rmtree(tdir)
-
-    return reqs
-
-
-def encode_requirements(requirement):
-    """Encode the project's module and version."""
-    # Default dependency base proxy url
-    base_url = "https://proxy.golang.org/"
-    # Encoding based on
-    # https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
-    encoded_module = ""
-    for char in requirement[0]:
-        if char.isupper():
-            encoded_module += "!" + char.lower()
-        else:
-            encoded_module += char
-    encoded_version = ""
-    for char in requirement[1]:
-        if char.isupper():
-            encoded_version += "!" + char.lower()
-        else:
-            encoded_version += char
-    return (encoded_module, encoded_version)
-
-
-def initialize_package(name, url, version, path, args):
-    """Setup package and try to build it."""
-    if already_built(name, version):
-        return True
-    print(f"Trying to autospecnew {name} - {version}")
-
-    proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
-    if f"{name} already exists at" in proc.stdout.decode("utf-8"):
-        return build(path, args, name, url, version)
-    return proc.returncode == 0
-
-
-def build_recursive(path, args, name, url, version=None, success=None):
-    """Try and recursively build packages."""
-    if not success:
-        # First package being built, common utilities already run
-        # so just use autospec directly
-        success = {}
-        ret = build(path, args, name, url, version)
-    else:
-        ret = initialize_package(name, url, version, path, args)
-
-    if ret:
-        # Completed so we are done!
-        return True
-
-    if not missing_dependencies(name):
-        # Failed but not due to missing dependencies, bail
-        print(f"Unknown error building {name} - {version}")
-        return False
-
-    reqs = get_dependencies(name)
-    if not reqs:
-        # Didn't try and add anything so don't need to build again
-        print(f"Build failed with no missing requirements: {name} - {version}")
-        return False
-
-    print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
-    for req in reqs:
-        ereq = encode_requirements(req)
-        rname = "go-" + req[0].replace("/", "-")
-        rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
-        if (rname, req[1]) in success:
-            # Rebuild detected, figure out if it is okay or not
-            if success[(rname, req[1])]:
-                # Already succeeded building req short circuit success
-                continue
-            else:
-                # Encountered a build loop, bail out as this likely
-                # requires manual fixing
-                print(f"Detected build loop when building {rname} - {req[1]}")
-                return True
-        success[(rname, req[1])] = False
-        if not build_recursive(path, args, rname, rurl, req[1], success):
-            print(f"Failed to build dependency {rname} - {req[1]}")
-            return False
-
-        success[(rname, req[1])] = True
-        subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
-        subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
-
-    # Create the repo with the new packages added
-    subprocess.run(["make", "localrepocreate"], capture_output=True)
-    # Retry previously failed build assuming dependencies got added
-    print(f"Rebuilding after resolving dependencies {name} - {version}")
-    ret = build(path, args, name, url, version)
-    print("")
-
-    return ret
-
-
-def main():
-    """Start program execution."""
-    args = parse_args()
-    path = os.path.dirname(os.path.realpath(__file__))
-    return build_recursive(path, args, args.name, args.url)
-
-if __name__ == '__main__':
-    if not main():
-        sys.exit(-1)
-    print("\n\nBuild completed\n\n")

licenses-extra

@@ -5,3 +5,4 @@ GFDL-1.3+
 MIT-Opengroup
 WXwindows
 w3c
+libpng-2.0

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:

vendor.py

@@ -9,9 +9,8 @@ import subprocess
 import tempfile
 import time
 
-from git import Repo
-
 import requests
+from git import Repo
 
 
 def vendor_check():
@@ -30,6 +29,7 @@ def vendor_check():
 
 def get_args():
     parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
     parser.add_argument('url')
     parser.add_argument('name')
     parser.add_argument('git')
@@ -44,85 +44,125 @@ def setup_content(url):
     with open(outfile, 'wb') as cfile:
         cfile.write(response.content)
 
-    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
     os.remove(outfile)
     return tdir
 
 
 def setup_cargo_vendor(path):
+    cargo_paths = []
     for dirpath, _, files in os.walk(path):
         for fname in files:
             if fname == "Cargo.toml":
-                return dirpath
-    return False
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
 
 
-def update_cargo_vendor(path, name, git):
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
     git_uri = os.path.join(git, name)
-    vendor_path = os.path.join(path, 'vendor')
-    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    vendor_path = os.path.join(tmpdir, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
     if not os.path.isdir(vendor_git):
         # initialize a git repo
-        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
-                       shell=True, check=True)
-    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
-    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
-                   shell=True, check=True)
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
-    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
-                   shell=True, check=True)
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
+                                   check=True, stdout=subprocess.PIPE,
+                                   universal_newlines=True).stdout
+    with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
+        gafile.write("* text=false\n")
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
-    if not len(repo.untracked_files) > 0 or repo.is_dirty():
-        return False
-    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
+        # Always use the newest tag as sometimes a new tag will
+        # be created but the package won't be updated to use it
+        # for a different failure reason.
+        tag = sorted(repo.tags, key=lambda x: x.name, reverse=True)[0]
+        return tag, cargo_vendors
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     gmt = time.gmtime()
     tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
     subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
-                   check=True)
+                   check=True, stdout=subprocess.DEVNULL)
     subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     time.sleep(30)
-    return tag
+    return tag, cargo_vendors
 
 
-def update_cargo_sources(name, tag):
+def update_cargo_sources(name, tag, cargo_vendors):
     makefile = []
-    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+    options = []
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
                                  'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.match(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
                 makefile.append(line)
     with open('Makefile', 'w', encoding='utf8') as mfile:
         mfile.writelines(makefile)
 
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+    with open('cargo_vendors', 'w', encoding='utf8') as cfile:
+        cfile.write(cargo_vendors)
+
 
 def main():
+    args = get_args()
+
     vtype = vendor_check()
     if not vtype:
+        print(args.archives)
         return
 
-    args = get_args()
     tdir = setup_content(args.url)
     if vtype == 'cargo':
-        vdir = setup_cargo_vendor(tdir)
-        if vdir:
-            tag = update_cargo_vendor(vdir, args.name, args.git)
-            if tag:
-                update_cargo_sources(args.name, tag)
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
+            print(args.archives)
+        else:
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
+            update_cargo_sources(args.name, tag, cargo_vendors)
     shutil.rmtree(tdir)
 
+
 if __name__ == '__main__':
     main()