Add support for multiple Cargo.toml files

Some projects store multiple Cargo.toml files, let vendor handle all
of them (might be worth having some config for certain Cargo.toml
paths to skip in the future though).

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -9,21 +9,26 @@ include $(TOPLVL)/projects/common/Makefile.shared
 
 SHELL  = /bin/bash
 
+CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+
 SPECFILE = $(addsuffix .spec, $(PKG_NAME))
 
-SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || rpmspec -D '_vendor clr' --srpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' $(SPECFILE))
+SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || $(call queryspec,%{NVR}\n,$(SPECFILE)))
 SRPMFILE = results/$(SRPMVERS).src.rpm
 
 LATEST_RPMS = $(wildcard rpms/*.rpm)
-DEBUGINFO_RPM = $(wildcard rpms/*-debuginfo-*.rpm)
+DEBUGINFO_RPM = $(wildcard rpms/${PKG_NAME}-debuginfo-$(shell $(call queryspec,%{V}-%{R},${SPECFILE})).*.rpm)
 RPMS ?= $(LATEST_RPMS)
 
-WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
+WITH_SUDO = sudo
 
 # MOCK_CONFIG_VAL is set in Makefile.shared
 MOCK_SMP_FLAGS=$(patsubst -j%, -D_smp_mflags\ -j%,$(filter -j%, $(MAKEFLAGS)))
 MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_SMP_FLAGS) $(MOCK_OPTS)
 
+MOCK_CLEANUP_OPT := $(if $(CLEANUP),--cleanup-after,--no-cleanup-after)
+AUTOSPEC_CLEANUP_OPT := $(if $(CLEANUP),-C)
+
 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
 __missingsources := $(filter-out $(wildcard ${__allsources}), ${__allsources})
@@ -46,7 +51,7 @@ upstream:
 #help spdxcheck: Checks that the package license is a valid SPDX license,
 #help or that it is in the exceptions list.
 spdxcheck: $(SPECFILE)
-	@for LICENSE in `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{LICENSE}\n' $(SPECFILE)`; do \
+	@for LICENSE in `$(call queryspec,%{LICENSE}\n,$(SPECFILE))`; do \
 		if ! grep -qx "$$LICENSE" $(TOPLVL)/projects/common/licenses-{spdx,exceptions,extra}; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
 	done; test -z "$$FAIL"
 
@@ -69,14 +74,14 @@ mockproper:
 	$(MOCK) --clean --uniqueext=$(PKG_NAME)
 
 configemail:
-	@test -d .git || git init
+	@test -d .git || git init -b main
 	@$(call subjectprefix,$(PKG_NAME))
 	@echo -n "Subject prefix email configured to: "
 	@git config --get format.subjectPrefix
 
 #help checkblacklist: Verify rpms files to not include banned files for Clear Linux OS.
 checkblacklist:
-	@$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist results/*.rpm
+	@$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(filter-out ${DEBUGINFO_RPM},${RPMS})
 
 prebuild-checks:
 
@@ -86,7 +91,7 @@ prebuild-checks:
 #help which creates a src.rpm file from the spec file.
 #help Use MOCK_OPTS environment varible to pass down arbitrary mock options.
 build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
-	$(MOCK) $(SRPMFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MOCK) $(SRPMFILE) --result=results/ $(MOCK_CLEANUP_OPT) --uniqueext=$(PKG_NAME)
 	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
@@ -94,7 +99,7 @@ build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
 
 #help build-nocheck: Same as 'make build', but do not run the package's test suite.
 build-nocheck: configemail $(SPECFILE) upstream $(SRPMFILE)
-	$(MOCK) $(SRPMFILE) --nocheck --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MOCK) $(SRPMFILE) --nocheck --result=results/ $(MOCK_CLEANUP_OPT) --uniqueext=$(PKG_NAME)
 	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
@@ -115,7 +120,7 @@ shell:
 .PHONY: $(SRPMFILE)
 $(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice clean-old-content
 	@$(MAKE) spdxcheck
-	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/ $(MOCK_CLEANUP_OPT) --uniqueext=$(PKG_NAME)
 	mv results/root.log results/srpm-root.log
 	mv results/build.log results/srpm-build.log
 
@@ -127,7 +132,7 @@ $(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice clean-old-content
 # initialized locally, or if the remote contains no commits.
 pullrebase:
 	@if [ -z "$$SKIP_GIT" ] && [ -e .git ]; then \
-		if ! git rev-parse --verify --quiet origin/master > /dev/null; then \
+		if ! git rev-parse --verify --quiet origin/main > /dev/null; then \
 			echo "Remote not yet initialized. Continuing."; \
 			exit 0; \
 		fi; \
@@ -141,7 +146,7 @@ pullrebase:
 			echo "Current branch up-to-date. Continuing."; \
 			exit 0; \
 		fi; \
-		echo "Rebasing to origin/master"; \
+		echo "Rebasing to origin/main"; \
 		if git status --porcelain | grep -q '^.[^?]'; \
 		then \
 			git stash save; \
@@ -161,10 +166,12 @@ preautospec-checks:
 #help to autospec.
 #help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: preautospec-checks pullrebase localreponotice clean-old-content
+	git rm --ignore-unmatch pumpAutospec || rm -f pumpAutospec
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
@@ -177,7 +184,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		$${SETVERSION:+ --version $${SETVERSION}} \
 		$${NON_INTERACTIVE:+ --non_interactive} \
 		$${SKIP_GIT:+ --skip-git} \
-		$${CLEANUP:+ -C} \
+		$(AUTOSPEC_CLEANUP_OPT) \
 		$(firstword $(NEWURL) $(URL));
 	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@$(MAKE) spdxcheck
@@ -212,17 +219,28 @@ autospecnostate:
 scanlicense:
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --config $(AUTOSPEC_CONF) --license-only $(firstword $(NEWURL) $(URL)) --name $(PKG_NAME)
 
-#help bump: Increments the release file by one.
+#help bump: Increments the package release number by one and commits the result. If
+#help the variable BUMP_MSG is set, its value is used as the commit summary.
+#help Otherwise a generic commit summary is used.
 bump:
+	git stash
 	git pull --rebase
 	$(MAKE) bumpnogit
 	git add $(SPECFILE) release
-	git commit -a -m "version bump from $(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)) to `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)`"
+	fromver=$(shell $(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))); \
+	tover=`$(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))`; \
+	if [[ -n "$(BUMP_MSG)" ]]; then \
+		bumpmsg="$(BUMP_MSG)"; \
+	else \
+		bumpmsg="version bump from $$fromver to $$tover"; \
+	fi; \
+	git commit -a -m "$$bumpmsg"
 
-#help bumpnogit: Increments the release file by one and does not commit changes.
+#help bumpnogit: Increments the package release number by one and does not commit
+#help changes.
 bumpnogit:
 	@$(MAKE) spdxcheck
-	oldrel=$(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{RELEASE}\n' $(SPECFILE)); \
+	oldrel=$(shell $(call queryspec,%{RELEASE}\n,$(SPECFILE))); \
 	newrel=$$(($$oldrel + 1)); \
 	sed -i "s/^\(Release[[:blank:]]*:[[:blank:]]*\)$$oldrel$$/\1$$newrel/" $(SPECFILE); \
 	echo "$$newrel" > release;
@@ -232,7 +250,7 @@ $(__missingsources): sources
 # First argument is a Clear Linux build number (e.g. 22000)
 define fetch-from-srpm
 	tmp=$$(mktemp -d -p "."); \
-	srpm=$$(rpmspec --srpm -q --qf "%{NVR}.src.rpm\n" $(SPECFILE)); \
+	srpm=$$($(call queryspec,%{NVR}.src.rpm\n,$(SPECFILE))); \
 	if cd "$$tmp" && ! curl -s -S -f -L -O $(MIRROR_CURL_OPTS) $(DOWNLOAD_MIRROR)/releases/$(1)/clear/source/SRPMS/$$srpm; then \
 		echo "Failed to download $$srpm from Clear Linux release $(1)." >&2; \
 		cd .. && rm -rf "$$tmp"; \
@@ -240,7 +258,7 @@ define fetch-from-srpm
 	fi; \
 	rpm2cpio $$srpm | cpio --quiet -id; \
 	cd ..; \
-	srclist=$$(rpmspec --srpm -q --qf "[%{SOURCE}\n]" $(SPECFILE)); \
+	srclist=$$($(call queryspec,[%{SOURCE}\n],$(SPECFILE))); \
 	for s in $$srclist; do \
 		if ! mv --no-clobber "$$tmp"/$$s .; then \
 			echo "Missing source file \"$$s\" in $${srpm}." >&2; \
@@ -279,7 +297,7 @@ else
 sources:
 	@$(MAKE) generateupstream; \
 	[ $$? -eq 0 ] && exit 0; \
-	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	nvr="$$($(call queryspec,%{NAME}\t%{VERSION}\t%{RELEASE}\n,$(SPECFILE)))"; \
 	git -C $(TOPLVL)/projects/common fetch --tags >/dev/null 2>&1; \
 	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
 	for b in $$latest_builds; do \
@@ -305,8 +323,8 @@ prekoji-checks:
 koji: prekoji-checks kojidef
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
-	@if [ "$$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then \
-		echo "Error: Must be on the master branch to submit to koji" >&2; \
+	@if [ "$(CURRENT_BRANCH)" != "main" ]; then \
+		echo "Error: Must be on the \"main\" branch to submit to koji" >&2; \
 		exit 1; \
 	fi; \
 	if ! git diff --quiet HEAD -- ${SPECFILE}; then \
@@ -314,7 +332,7 @@ koji: prekoji-checks kojidef
 		exit 1; \
 	fi; \
 	git fetch; \
-	if git rev-parse --verify --quiet origin/master >/dev/null; then \
+	if git rev-parse --verify --quiet origin/main >/dev/null; then \
 		git pull --rebase; \
 	fi; \
 	if git ls-remote --tags --exit-code origin $(SRPMVERS) >/dev/null; then \
@@ -322,8 +340,15 @@ koji: prekoji-checks kojidef
 		exit 1; \
 	fi
 	git tag $(SRPMVERS)
-	git push origin master refs/tags/$(SRPMVERS)
+	git push origin $(CURRENT_BRANCH):main refs/tags/$(SRPMVERS)
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
+	@if [ -f bump.list ]; then \
+		$(MAKE) koji-waitrepo; \
+		msg="Bump for $(shell $(call queryspec,%{NAME} %{VERSION}\n,$(SPECFILE)))"; \
+		for bump_dep in $$(cat bump.list); do \
+			(cd ../"$$bump_dep" && $(MAKE) bump BUMP_MSG="$$msg" && $(MAKE) koji-nowait) \
+		done \
+	fi
 
 #help rekoji: In case a previous 'make koji' failed, trigger a rebuild for the
 #help same tag.
@@ -347,6 +372,58 @@ koji-nowait:
 rekoji-nowait:
 	$(MAKE) KOJI_NOWAIT="--nowait --background" rekoji
 
+#help koji-waitrepo: Wait for current package changes to become available in the
+#help koji repo. If PKG_NAME is overridden on the command line, wait on changes for
+#help that package instead. This capability allows the developer to "chain" builds
+#help for submission to koji according to the state of the package repos on their
+#help system. Additional options for `koji wait-repo` can be specified via the
+#help WAIT_OPTS variable.
+koji-waitrepo: kojidef
+	@if [[ ! "${PKG_NAME}" =~ ^[A-Za-z0-9._+-]+$$ ]]; then \
+		echo "[ERROR] Invalid package name \"${PKG_NAME}\""; \
+		exit 1; \
+	fi; \
+	if [[ "${PKG_NAME}" != "$(notdir ${CURDIR})" ]]; then \
+		$(MAKE) -s -C ../${PKG_NAME} koji-waitrepo; \
+	else \
+		koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
+	fi
+
+.PHONY: koji-tag
+#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
+koji-tag:
+	koji tag-pkg dist-clear $(SRPMVERS)
+
+.PHONY: koji-untag
+#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
+koji-untag:
+	koji untag-pkg dist-clear $(SRPMVERS)
+
+
+update-versions:
+
+.PHONY: update
+#help update: Tries to run update.sh if it exists. update.sh is expected
+#help to check for version updates, exit (successfully) if none are found
+#help or update the package and push the update to koji.
+#help If no update.sh exists, update the version, autospec and push the
+#help update to koji (bumping the bump.list if found).
+update: export AUTOSPEC_UPDATE=1
+update:
+	@if [ -f update.first ]; then \
+		for pkg in $$(< update.first); do \
+			(cd "../$${pkg}" || exit 1; $(MAKE) update-versions || exit 0; $(MAKE) update) || exit; \
+		done; \
+	fi
+	@if [ -f update.sh ]; then \
+		./update.sh; \
+	else \
+		$(MAKE) -s update-versions && \
+		$(MAKE) autospec CLEANUP=1 && \
+		$(MAKE) -s koji && \
+		$(MAKE) koji-waitrepo; \
+	fi \
+
 logcheck:
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 
@@ -368,21 +445,6 @@ repoadd: $(TOPLVL)/repo
 	$(MAKE) -s localrepocreate
 	$(MAKE) -s localrepoenable
 
-#help repostage: Stages locally-built RPMs for this package to the local RPM
-#help directory. If this directory does not yet exist, it is created and
-#help enabled.
-repostage: $(TOPLVL)/repo
-	@if [ -z "${LATEST_RPMS}" ]; then \
-		echo "No rpms found in rpms/ directory."; \
-		exit 1; \
-	fi
-	$(MAKE) -s repodel NO_CREATEREPO=1
-	@for r in ${LATEST_RPMS}; do \
-		echo "+$${rpm#rpms/}"; \
-		echo "$${rpm#rpms/}" >> .repo-index; \
-	done; \
-	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
-
 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
 repodel: $(TOPLVL)/repo
@@ -449,6 +511,10 @@ install-debuginfo-local:
 		sudo chown -R dbginfo:dbginfo "$$d"; \
 		sudo cp -a "$$d" $$dest/; \
 	done; \
+	find $$tmpdir/usr/share/debug/.build-id -type l 2> /dev/null | while read -r link; do \
+		new_target=$$(readlink $$link | sed 's|/usr/share/debug|/usr/lib/debug|'); \
+		sudo ln -sf "$$new_target" "$$link"; \
+	done; \
 	find $$tmpdir/usr/share/debug/ -mindepth 1 -maxdepth 1 -regextype awk -regex '.*/(.build-id|boot|lib|sbin|usr)$$' 2> /dev/null | while read -r d; do \
 		sudo chown -R dbginfo:dbginfo "$$d"; \
 		sudo cp -a "$$d" $$dest/; \
@@ -498,7 +564,7 @@ generateupstream:
 				exit 1; \
 			fi; \
 		fi; \
-		echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
+		echo $$(sha1sum -- $$filename | cut -d\  -f1)/$$filename >> upstream; \
 	done
 	@rm -f upstream.bak
 	@cat upstream
@@ -516,6 +582,44 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+.PHONY: whatrequires
+#help whatrequires: Output a list of packages that directly depend on this one,
+#help showing the subpackage-level breakdown. Each line of output has the format
+#help "SUBPACKAGE(|SYMBOL)? <- PACKAGE (ARCH)". If ARCH is "src", the meaning is
+#help "PACKAGE has a BuildRequires (build dependency) on SUBPACKAGE". And if ARCH
+#help is "x86_64", the meaning is "PACKAGE has a Requires (runtime dependency) on
+#help SUBPACKAGE". The optional "|SYMBOL" portion is printed when the symbol
+#help required does not match the subpackage name. Note that the ability to query
+#help version-qualified dependencies is not yet implemented... (For example, if
+#help python3-core provides the version-qualified symbol "python(abi) = 3.9",
+#help running `make whatrequires` for `python3` will detect packages that depend on
+#help "python(abi)", but not "python(abi) = 3.9".)
+whatrequires:
+	@Q="dnf --config=${PM_CONF} repoquery --quiet --releasever=clear"; \
+	TMP=$$(mktemp -d); trap "rm -rf $$TMP" EXIT; \
+	$${Q} --provides ${PKG_NAME} | awk '{ print $$1 }' > $$TMP/${PKG_NAME}; \
+	( \
+	while read -r provide; do \
+		$${Q} --qf="${PKG_NAME} <- %{NAME} (x86_64)" --arch=x86_64 --srpm --whatrequires $$provide; \
+		$${Q} --qf="${PKG_NAME} <- %{NAME} (src)" --arch=src --whatrequires $$provide; \
+	done < $$TMP/${PKG_NAME} \
+	) | awk '$$3 != "${PKG_NAME}"' | LC_COLLATE=C sort -u; \
+	sed -n 's/^%package *\(.*\)$$/\1/p' ${PKG_NAME}.spec | sort > $$TMP/subpkgs; \
+	while read -r val; do \
+		if grep -qE '^-n +' <<< $$val; then \
+			subpkg=$$(awk '{ print $$2 }' <<< $$val); \
+		else \
+			subpkg=${PKG_NAME}-$$val; \
+		fi; \
+		$${Q} --provides $$subpkg | awk '{ print $$1 }' > $$TMP/$$subpkg; \
+		( \
+		while read -r provide; do \
+			$${Q} --qf="$$subpkg|$$provide <- %{NAME} (x86_64)" --arch=x86_64 --srpm --whatrequires $$provide; \
+			$${Q} --qf="$$subpkg|$$provide <- %{NAME} (src)" --arch=src --whatrequires $$provide; \
+		done < $$TMP/$$subpkg \
+		) | awk '$$3 != "${PKG_NAME}"' | sed "s/^$$subpkg|\($$subpkg\)/\1/" | LC_COLLATE=C sort -u; \
+	done < $$TMP/subpkgs
+
 # Define LTS-specific targets in a separate makefile
 -include $(TOPLVL)/projects/common/Makefile.common.lts
 

Makefile.config

@@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs
 # Location where project repos are hosted.
 PRJ_BASE_URL = https://github.com/clearlinux
 
+# Location where vendor repos are hosted.
+VND_BASE_URL =
+
 # Project repos that are integral to the common tooling.
 PROJ_PKGS = autospec clr-bundles common
 

Makefile.shared

@@ -47,6 +47,14 @@ define subjectprefix
 	git config format.subjectPrefix "PATCH $(1)"
 endef
 
+# Query the spec file (or spec files) listed in argument 2 for the field(s)
+# specified in argument 1. The fields are derived from the source rpm(s)
+# generated by the specs. Queried fields that are declared within the spec file
+# must be declared above the first %files section.
+define queryspec
+rpmspec --define='_vendor clr' --srpm --query --queryformat='$(1)' <(sed -n '1,/%files/p' $(2))
+endef
+
 # If GITOLITE_BASE_URL is defined, sets a repo's push URL for that gitolite
 # instance. Accepts one argument: the path to the repo on gitolite. If
 # GITOLITE_BASE_URL is not defined, no push URL is set -- the shell's null

Makefile.toplevel

@@ -31,11 +31,13 @@ $(proj_PKGS):
 	@proj=$(patsubst proj_%,%,$@); \
 	[ -d projects/$$proj ] || ( \
 	echo "Checking out: projects/$$proj"; \
-	git clone $(PRJ_BASE_URL)/$$proj projects/$$proj; \
-	cd projects/$$proj; \
 	if [ "$$proj" = "autospec" ] ; then \
-		git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
+		git clone https://github.com/clearlinux/autospec.git projects/autospec; \
 	else \
+		git clone $(PRJ_BASE_URL)/$$proj projects/$$proj; \
+	fi; \
+	cd projects/$$proj; \
+	if [ "$$proj" != "autospec" ] ; then \
 		$(call gitoliteurl,projects/$$proj); \
 	fi; \
 	$(call subjectprefix,$$proj); \
@@ -50,7 +52,7 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	remotepkg=$(call remotepkgname,$$pkg); \
 	[ -d packages/$$pkg ] || ( \
 	echo "Checking out: packages/$$pkg"; \
-	git clone $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
+	git clone -b main $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
 	cd packages/$$pkg; \
 	$(call gitoliteurl,packages/$$pkg); \
 	$(call subjectprefix,$$pkg); \
@@ -66,19 +68,22 @@ ${PULL_PKGS}:
 	if [ ! -d "$$p/.git" ]; then echo "Nothing to pull for $$p - $$p/.git missing"; exit 0; fi; \
 	cd "$$p" ; \
 	if git remote | grep origin >/dev/null 2>&1; then \
-		O=$$(git rev-parse origin/master 2>/dev/null); \
+		case "$$p" in \
+			("projects/"*) BRANCH=master ;; \
+			("packages/"*) BRANCH=main ;; \
+			(*) BRANCH=master ;; \
+		esac; \
+		O=$$(git rev-parse origin/"$$BRANCH" 2>/dev/null); \
 		git fetch --tags origin >/dev/null 2>&1; \
-		N=$$(git rev-parse origin/master); \
+		N=$$(git rev-parse origin/"$$BRANCH"); \
 		if [ "$$O" != "$$N" ]; then \
 			echo "Updating: $$p"; \
-			if ! git merge --ff-only origin/master 2>/dev/null; then \
-				echo "$$p: Cannot fast-forward $$(git rev-parse --abbrev-ref HEAD) to origin/master" ; \
+			if ! git merge --ff-only origin/"$$BRANCH" 2>/dev/null; then \
+				echo "$$p: Cannot fast-forward $$(git rev-parse --abbrev-ref HEAD) to origin/$$BRANCH" ; \
 			fi ; \
 			git --no-pager log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit $$O..$$N; echo; \
 		fi ; \
-		if [ "$$p" = "projects/autospec" ] ; then \
-			git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
-		else \
+		if [ "$$p" != "projects/autospec" ] ; then \
 			$(call gitoliteurl,$$p); \
 		fi ; \
 	else \
@@ -121,11 +126,17 @@ endif
 pull: ${PULL_PKGS}
 
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
-#help removes the associated package repos from the ./packages tree
+#help removes the associated package repos from the ./packages tree and stores
+#help it in packages_old. If it already exists in packages_old, skip it.
 clean-pkgs-dir: $(PACKAGES_FILE)
 	@for p in ${PKGS_ORPHANED}; do \
-		echo "Removing $$p from packages, it is no longer in common/packages."; \
-		rm -rf packages/$$p; \
+		mkdir -p packages_old; \
+		if [ -d "packages_old/$$p" ]; then \
+			echo "packages_old/$$p already exists, not touching it"; \
+		else \
+			echo "Moving $$p from packages to packages_old, as it is no longer in common/packages."; \
+			mv packages/$$p packages_old/$$p; \
+		fi; \
 	done
 	@echo "packages directory cleaned";
 
@@ -159,32 +170,32 @@ status: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	if [ -d "$$p/.git" ] &&  [ -n "$$(git -C $$p status -uno --porcelain)" ]; then echo "Uncommitted changes in $$p:"; git -C "$$p" status --short; fi ;\
 	done
 
-#help diff: Runs git diff for all package repos and displays the output using
-#help diffstat. Requires that diffstat is installed.
+#help diff: Runs git diff for all package repos and displays the diffstat using
+#help the --stat flag.
 diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
-		(cd $$p ; git status | grep -q 'nothing to commit, working [^ ]* clean' || (echo "Uncommitted changes in: $$p"; git diff |diffstat -p1)) ;\
+		(cd $$p; git status --porcelain | grep -vq '^?? ' && (echo "Uncommitted changes in: $$p"; git --no-pager diff --stat); :) ;\
 	done
 
-#help log: Run git log origin/master..HEAD for all package repos, thus
+#help log: Run git log origin/main..HEAD for all package repos, thus
 #help displaying any committed but unpushed changes.
 log: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
-		( cd $$p; if [ "$$(git rev-parse --verify HEAD)" != "$$(git rev-parse --verify origin/master)" ] ; then \
-		echo $$p ; git --no-pager log --pretty="format:%s" origin/master..HEAD ; echo ; echo;\
+		( cd $$p; if [ -n "$$(git rev-list origin/main..HEAD)" ] ; then \
+		echo $$p ; git --no-pager log --pretty="format:%s" origin/main..HEAD ; echo ; echo;\
 		fi ; ) \
 	done
 
 #help versions: Displays the version of each package in the ./packages tree.
 versions: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
-		(cd $$p; if ls *.spec &> /dev/null; then rpmspec -q --queryformat '%{NAME}-%{VERSION}\n' *.spec | head -n1; fi) ;\
+		(cd $$p; if ls *.spec &> /dev/null; then $(call queryspec,%{NAME}-%{VERSION}\n,*.spec) | head -n1; fi) ;\
 	done
 
 #help releases: Like 'make versions', but also displays the release number.
 releases: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
-		(cd $$p; if ls *.spec &> /dev/null; then rpmspec -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' *.spec | head -n1; fi) ;\
+		(cd $$p; if ls *.spec &> /dev/null; then $(call queryspec,%{NVR}\n,*.spec) | head -n1; fi) ;\
 	done
 
 #help provides:     Is used to find out which RPM package provides some file.
@@ -211,7 +222,7 @@ autospecnew: preautospecnew-checks localreponotice
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
 		( \
 			cd $(TOPLVL)/packages/$(NAME); \
-			git init; \
+			git init -b main; \
 			git remote add origin $(PKG_BASE_URL)/$(NAME); \
 			$(call gitoliteurl,packages/$(NAME)); \
 			$(call subjectprefix,$(NAME)); \
@@ -226,7 +237,7 @@ autospecnew: preautospecnew-checks localreponotice
 			--mock-config $(MOCK_CONFIG_VAL) \
 			--mock-opts="$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
-			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+			${NON_INTERACTIVE} ${SKIP_GIT} $(if $(CLEANUP),-C) \
 			$(URL); \
 		if [ $$? -eq 0 ]; then \
 			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
@@ -245,51 +256,6 @@ autospecnew: preautospecnew-checks localreponotice
 		exit 1; \
 	fi
 
-#help autospecnewgo: Creates a new autospec package with (dependencies)
-#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
-#help autospec will be created in the help process.
-#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
-#help to autospec.
-#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
-autospecnewgo: preautospecnew-checks localreponotice
-	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
-		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
-		exit 1; \
-	fi
-	-$(MAKE) clone_$(NAME)
-	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
-		echo "no remote repository found, creating new package repository and running autospec"; \
-		mkdir -p $(TOPLVL)/packages/$(NAME); \
-		( \
-			cd $(TOPLVL)/packages/$(NAME); \
-			git init; \
-			git remote add origin $(PKG_BASE_URL)/$(NAME); \
-			$(call gitoliteurl,packages/$(NAME)); \
-			$(call subjectprefix,$(NAME)); \
-		); \
-		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
-		python3 $(TOPLVL)/projects/common/go-builder.py \
-			--target packages/$(NAME) \
-			--integrity \
-			--config "$(AUTOSPEC_CONF)" \
-			--name $(NAME) \
-			--archives $(ARCHIVES) \
-			--mock-config $(MOCK_CONFIG_VAL) \
-			--mock-opts "$(MOCK_OPTS)" \
-			$(URL); \
-		if [ $$? -eq 0 ]; then \
-			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
-			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
-			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
-			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
-		else \
-			exit 1; \
-		fi; \
-	else \
-		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
-		exit 1; \
-	fi
-
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
 repoenable: localrepoenable ;

README.md

@@ -23,7 +23,7 @@ Download the [user setup](user-setup.sh) script and run it on your Clear Linux
 system as an unprivileged user.
 
 ```
-$ curl -O https://raw.githubusercontent.com/clearlinux/common/master/user-setup.sh
+$ curl -O https://raw.githubusercontent.com/clearlinux/common/main/user-setup.sh
 $ chmod +x user-setup.sh
 $ ./user-setup.sh
 ```

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

conf/clear.cfg

@@ -4,14 +4,14 @@ config_opts['legal_host_arches'] = ('x86_64',)
 config_opts['chroot_setup_cmd'] = 'groupinstall build srpm-build'
 config_opts['dist'] = 'clear'  # only useful for --resultdir variable subst
 config_opts['extra_chroot_dirs'] = [ '/run/lock', ]
-config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s  %(user)s'
+config_opts['useradd'] = '/usr/sbin/useradd -m -u {{chrootuid}} -g {{chrootgid}} -d {{chroothome}}  {{chrootuser}}'
 config_opts['plugin_conf']['ccache_enable'] = False
 config_opts['plugin_conf']['root_cache_opts']['compress_program'] = 'zstd -5'
 config_opts['plugin_conf']['root_cache_opts']['extension'] = '.zst'
 config_opts['releasever'] = 'clear'
 config_opts['package_manager'] = 'dnf'
 config_opts['chrootgroup'] = 'mockbuild'
-config_opts['use_bootstrap_container'] = False
+config_opts['use_bootstrap'] = False
 
 
 config_opts['dnf.conf'] = """

go-builder.py

@@ -1,285 +0,0 @@
-#!/usr/bin/env python3
-#
-# gowrap.py - part of autospec
-# Copyright (C) 2019 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import argparse
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-
-
-def parse_args():
-    """Parse commandline arguments."""
-    parser = argparse.ArgumentParser()
-    parser.add_argument("url", help="Location of package archive")
-    parser.add_argument("-t", "--target", dest="target", action="store",
-                        default=None,
-                        help="Target location to create or reuse")
-    parser.add_argument("-c", "--config", dest="config", action="store",
-                        default="/usr/share/defaults/autospec/autospec.conf",
-                        help="Set configuration file to use")
-    parser.add_argument("-n", "--name", action="store", dest="name", default="",
-                        help="Override the package name")
-    parser.add_argument("-i", "--integrity", action="store_true",
-                        default=False,
-                        help="Search for package signature from source URL and "
-                             "attempt to verify package")
-    parser.add_argument("-m", "--mock-config", action="store", default="clear",
-                        help="Value to pass with Mock's -r option. Defaults to "
-                             "\"clear\", meaning that Mock will use "
-                             "/etc/mock/clear.cfg.")
-    parser.add_argument("-o", "--mock-opts", action="store", default="",
-                        help="Arbitrary options to pass down to mock when "
-                        "building a package.")
-    parser.add_argument('-a', "--archives", action="store",
-                        dest="archives", default=[], nargs='*',
-                        help="tarball URLs for additional source archives and"
-                        " a location for the sources to be extacted to (e.g."
-                        " http://example.com/downloads/dependency.tar.gz"
-                        " /directory/relative/to/extract/root )")
-
-    return parser.parse_args()
-
-
-def missing_dependencies(name):
-    """Check if the build failed due to missing dependencies."""
-    root_log = os.path.join("packages", name, "results", "root.log")
-    if not os.path.exists(root_log):
-        return False
-    with open(root_log, "r") as lfile:
-        for line in lfile.readlines():
-            if "No matching package to install" in line:
-                return True
-    return False
-
-
-def already_built(name, version):
-    """Check if the package and version are already built."""
-    if not os.path.exists(os.path.join("packages", name, "rpms")):
-        return False
-    vpath = os.path.join("packages", name, "versions")
-    if not os.path.exists(vpath):
-        return False
-    with open(vpath, "r") as vfile:
-            if version not in [x.strip() for x in vfile.readlines()]:
-                return False
-    return True
-
-
-def build(path, args, name, url, version):
-    """Try and build a package."""
-    if version:
-        ver_arg = ["-v", version]
-    else:
-        ver_arg = []
-    if already_built(name, version):
-        return True
-    print(f"Trying to autospec {name} - {version}")
-
-    proc = subprocess.run(["python3",
-                           f"{path}/../autospec/autospec/autospec.py", url,
-                           "-c", args.config, "-t", f"packages/{name}",
-                           "-n", name, "-m", args.mock_config,
-                           "-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
-    if proc.returncode == 0:
-        subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
-                       capture_output=True)
-    return proc.returncode == 0
-
-
-def parse_go_mod(path):
-    """Parse go.mod file for build requirements.
-
-    File content looks as follows:
-
-    module example.com/foo/bar
-
-    require (
-        github.com/BurntSushi/toml v0.3.1
-        git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
-        github.com/inconshreveable/mousetrap v1.0.0 // indirect
-        "github.com/spf13/cobra" v0.0.3
-        github.com/spf13/pflag v1.0.3 // indirect
-    )
-
-    Need to handle all require lines including //indirect.
-    Skip requires that use .git for now. May need to be handled
-    differently.
-    """
-    reqs = []
-    with open(path, "r") as gfile:
-        dep_start = False
-        for line in gfile.readlines():
-            # Ideally the mod file is generated and the format is
-            # always correct but add a few defenses just in case
-            line = line.strip()
-            if line.startswith("//"):
-                # Skip comments
-                continue
-            if dep_start:
-                # End of the require section
-                if line.startswith(")"):
-                    break
-                req = line.split()[:2]
-                req[0] = req[0].replace('"', '')
-                if req[0].endswith(".git"):
-                    continue
-                reqs.append(req)
-                continue
-            if line.startswith("require ("):
-                dep_start = True
-    return reqs
-
-
-def get_dependencies(name):
-    """Return path to the go.mod file if it exists."""
-    command = None
-    path = None
-    pdir = os.path.join("packages", name)
-    reqs = []
-    for fname in os.listdir(pdir):
-        # Find the archive
-        if fname.endswith("zip"):
-            command = ["unzip", fname]
-        elif ".tar." in fname:
-            command = ["tar", "xf", fname]
-        else:
-            command = None
-
-        if command:
-            # Decompress archive
-            tdir = tempfile.mkdtemp()
-            tfile = os.path.join(tdir, fname)
-            shutil.copyfile(os.path.join(pdir, fname), tfile)
-            proc = subprocess.run(command, cwd=tdir, capture_output=True)
-            if not proc.returncode:
-                # inspect contents for a "go.sum" file
-                for root, _, files in os.walk(tdir):
-                    if "go.mod" in files:
-                        path = f"{name}.gomod"
-                        shutil.copyfile(os.path.join(root, "go.mod"), path)
-                        reqs += parse_go_mod(path)
-                        os.unlink(path)
-            shutil.rmtree(tdir)
-
-    return reqs
-
-
-def encode_requirements(requirement):
-    """Encode the project's module and version."""
-    # Default dependency base proxy url
-    base_url = "https://proxy.golang.org/"
-    # Encoding based on
-    # https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
-    encoded_module = ""
-    for char in requirement[0]:
-        if char.isupper():
-            encoded_module += "!" + char.lower()
-        else:
-            encoded_module += char
-    encoded_version = ""
-    for char in requirement[1]:
-        if char.isupper():
-            encoded_version += "!" + char.lower()
-        else:
-            encoded_version += char
-    return (encoded_module, encoded_version)
-
-
-def initialize_package(name, url, version, path, args):
-    """Setup package and try to build it."""
-    if already_built(name, version):
-        return True
-    print(f"Trying to autospecnew {name} - {version}")
-
-    proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
-    if f"{name} already exists at" in proc.stdout.decode("utf-8"):
-        return build(path, args, name, url, version)
-    return proc.returncode == 0
-
-
-def build_recursive(path, args, name, url, version=None, success=None):
-    """Try and recursively build packages."""
-    if not success:
-        # First package being built, common utilities already run
-        # so just use autospec directly
-        success = {}
-        ret = build(path, args, name, url, version)
-    else:
-        ret = initialize_package(name, url, version, path, args)
-
-    if ret:
-        # Completed so we are done!
-        return True
-
-    if not missing_dependencies(name):
-        # Failed but not due to missing dependencies, bail
-        print(f"Unknown error building {name} - {version}")
-        return False
-
-    reqs = get_dependencies(name)
-    if not reqs:
-        # Didn't try and add anything so don't need to build again
-        print(f"Build failed with no missing requirements: {name} - {version}")
-        return False
-
-    print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
-    for req in reqs:
-        ereq = encode_requirements(req)
-        rname = "go-" + req[0].replace("/", "-")
-        rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
-        if (rname, req[1]) in success:
-            # Rebuild detected, figure out if it is okay or not
-            if success[(rname, req[1])]:
-                # Already succeeded building req short circuit success
-                continue
-            else:
-                # Encountered a build loop, bail out as this likely
-                # requires manual fixing
-                print(f"Detected build loop when building {rname} - {req[1]}")
-                return True
-        success[(rname, req[1])] = False
-        if not build_recursive(path, args, rname, rurl, req[1], success):
-            print(f"Failed to build dependency {rname} - {req[1]}")
-            return False
-
-        success[(rname, req[1])] = True
-        subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
-        subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
-
-    # Create the repo with the new packages added
-    subprocess.run(["make", "localrepocreate"], capture_output=True)
-    # Retry previously failed build assuming dependencies got added
-    print(f"Rebuilding after resolving dependencies {name} - {version}")
-    ret = build(path, args, name, url, version)
-    print("")
-
-    return ret
-
-
-def main():
-    """Start program execution."""
-    args = parse_args()
-    path = os.path.dirname(os.path.realpath(__file__))
-    return build_recursive(path, args, args.name, args.url)
-
-if __name__ == '__main__':
-    if not main():
-        sys.exit(-1)
-    print("\n\nBuild completed\n\n")

licenses-extra

@@ -5,3 +5,4 @@ GFDL-1.3+
 MIT-Opengroup
 WXwindows
 w3c
+libpng-2.0

licenses-spdx

@@ -16,6 +16,7 @@ AMDPLPA
 AML
 AMPAS
 ANTLR-PD
+ANTLR-PD-fallback
 APAFML
 APL-1.0
 APSL-1.0
@@ -30,6 +31,8 @@ Aladdin
 Apache-1.0
 Apache-1.1
 Apache-2.0
+App-s2p
+Arphic-1999
 Artistic-1.0
 Artistic-1.0-Perl
 Artistic-1.0-cl8
@@ -39,62 +42,92 @@ BSD-2-Clause
 BSD-2-Clause-FreeBSD
 BSD-2-Clause-NetBSD
 BSD-2-Clause-Patent
+BSD-2-Clause-Views
 BSD-3-Clause
 BSD-3-Clause-Attribution
 BSD-3-Clause-Clear
 BSD-3-Clause-LBNL
+BSD-3-Clause-Modification
+BSD-3-Clause-No-Military-License
 BSD-3-Clause-No-Nuclear-License
 BSD-3-Clause-No-Nuclear-License-2014
 BSD-3-Clause-No-Nuclear-Warranty
 BSD-3-Clause-Open-MPI
 BSD-4-Clause
+BSD-4-Clause-Shortened
 BSD-4-Clause-UC
 BSD-Protection
 BSD-Source-Code
 BSL-1.0
+BUSL-1.1
+Baekmuk
 Bahyph
 Barr
 Beerware
 BitTorrent-1.0
 BitTorrent-1.1
+Bitstream-Vera
 BlueOak-1.0.0
 Borceux
+C-UDA-1.0
+CAL-1.0
+CAL-1.0-Combined-Work-Exception
 CATOSL-1.1
 CC-BY-1.0
 CC-BY-2.0
 CC-BY-2.5
+CC-BY-2.5-AU
 CC-BY-3.0
+CC-BY-3.0-AT
+CC-BY-3.0-DE
+CC-BY-3.0-IGO
+CC-BY-3.0-NL
+CC-BY-3.0-US
 CC-BY-4.0
 CC-BY-NC-1.0
 CC-BY-NC-2.0
 CC-BY-NC-2.5
 CC-BY-NC-3.0
+CC-BY-NC-3.0-DE
 CC-BY-NC-4.0
 CC-BY-NC-ND-1.0
 CC-BY-NC-ND-2.0
 CC-BY-NC-ND-2.5
 CC-BY-NC-ND-3.0
+CC-BY-NC-ND-3.0-DE
+CC-BY-NC-ND-3.0-IGO
 CC-BY-NC-ND-4.0
 CC-BY-NC-SA-1.0
 CC-BY-NC-SA-2.0
+CC-BY-NC-SA-2.0-FR
+CC-BY-NC-SA-2.0-UK
 CC-BY-NC-SA-2.5
 CC-BY-NC-SA-3.0
+CC-BY-NC-SA-3.0-DE
+CC-BY-NC-SA-3.0-IGO
 CC-BY-NC-SA-4.0
 CC-BY-ND-1.0
 CC-BY-ND-2.0
 CC-BY-ND-2.5
 CC-BY-ND-3.0
+CC-BY-ND-3.0-DE
 CC-BY-ND-4.0
 CC-BY-SA-1.0
 CC-BY-SA-2.0
+CC-BY-SA-2.0-UK
+CC-BY-SA-2.1-JP
 CC-BY-SA-2.5
 CC-BY-SA-3.0
+CC-BY-SA-3.0-AT
+CC-BY-SA-3.0-DE
 CC-BY-SA-4.0
 CC-PDDC
 CC0-1.0
 CDDL-1.0
 CDDL-1.1
+CDL-1.0
 CDLA-Permissive-1.0
+CDLA-Permissive-2.0
 CDLA-Sharing-1.0
 CECILL-1.0
 CECILL-1.1
@@ -104,53 +137,79 @@ CECILL-B
 CECILL-C
 CERN-OHL-1.1
 CERN-OHL-1.2
+CERN-OHL-P-2.0
+CERN-OHL-S-2.0
+CERN-OHL-W-2.0
 CNRI-Jython
 CNRI-Python
 CNRI-Python-GPL-Compatible
+COIL-1.0
 CPAL-1.0
 CPL-1.0
 CPOL-1.02
 CUA-OPL-1.0
 Caldera
 ClArtistic
+Community-Spec-1.0
 Condor-1.1
 Crossword
 CrystalStacker
 Cube
 D-FSL-1.0
+DL-DE-BY-2.0
 DOC
+DRL-1.0
 DSDP
 Dotseqn
 ECL-1.0
 ECL-2.0
 EFL-1.0
 EFL-2.0
+EPICS
 EPL-1.0
 EPL-2.0
 EUDatagrid
 EUPL-1.0
 EUPL-1.1
 EUPL-1.2
+Elastic-2.0
 Entessa
 ErlPL-1.1
 Eurosym
+FDK-AAC
 FSFAP
 FSFUL
 FSFULLR
+FSFULLRWD
 FTL
 Fair
 Frameworx-1.0
+FreeBSD-DOC
 FreeImage
+GD
 GFDL-1.1
+GFDL-1.1-invariants-only
+GFDL-1.1-invariants-or-later
+GFDL-1.1-no-invariants-only
+GFDL-1.1-no-invariants-or-later
 GFDL-1.1-only
 GFDL-1.1-or-later
 GFDL-1.2
+GFDL-1.2-invariants-only
+GFDL-1.2-invariants-or-later
+GFDL-1.2-no-invariants-only
+GFDL-1.2-no-invariants-or-later
 GFDL-1.2-only
 GFDL-1.2-or-later
 GFDL-1.3
+GFDL-1.3-invariants-only
+GFDL-1.3-invariants-or-later
+GFDL-1.3-no-invariants-only
+GFDL-1.3-no-invariants-or-later
 GFDL-1.3-only
 GFDL-1.3-or-later
 GL2PS
+GLWTPL
 GPL-1.0
 GPL-1.0+
 GPL-1.0-only
@@ -175,7 +234,9 @@ Glide
 Glulxe
 HPND
 HPND-sell-variant
+HTMLTIDY
 HaskellReport
+Hippocratic-2.1
 IBM-pibs
 ICU
 IJG
@@ -190,7 +251,9 @@ Intel-ACPI
 Interbase-1.0
 JPNIC
 JSON
+Jam
 JasPer-2.0
+Knuth-CTAN
 LAL-1.2
 LAL-1.3
 LGPL-2.0
@@ -213,6 +276,8 @@ LPPL-1.1
 LPPL-1.2
 LPPL-1.3a
 LPPL-1.3c
+LZMA-SDK-9.11-to-9.20
+LZMA-SDK-9.22
 Latex2e
 Leptonica
 LiLiQ-P-1.1
@@ -220,31 +285,43 @@ LiLiQ-R-1.1
 LiLiQ-Rplus-1.1
 Libpng
 Linux-OpenIB
+Linux-man-pages-copyleft
 MIT
 MIT-0
 MIT-CMU
+MIT-Modern-Variant
 MIT-advertising
 MIT-enna
 MIT-feh
+MIT-open-group
 MITNFA
 MPL-1.0
 MPL-1.1
 MPL-2.0
 MPL-2.0-no-copyleft-exception
+MS-LPL
 MS-PL
 MS-RL
 MTLL
 MakeIndex
+Minpack
 MirOS
 Motosoto
 MulanPSL-1.0
+MulanPSL-2.0
 Multics
 Mup
+NAIST-2003
 NASA-1.3
 NBPL-1.0
+NCGL-UK-2.0
 NCSA
 NGPL
+NICTA-1.0
+NIST-PD
+NIST-PD-fallback
 NLOD-1.0
+NLOD-2.0
 NLPL
 NOSL
 NPL-1.0
@@ -260,6 +337,7 @@ Newsletr
 Nokia
 Noweb
 Nunit
+O-UDA-1.0
 OCCT-PL
 OCLC-2.0
 ODC-By-1.0
@@ -270,6 +348,8 @@ OFL-1.0-no-RFN
 OFL-1.1
 OFL-1.1-RFN
 OFL-1.1-no-RFN
+OGC-1.0
+OGDL-Taiwan-1.0
 OGL-Canada-2.0
 OGL-UK-1.0
 OGL-UK-2.0
@@ -293,6 +373,7 @@ OLDAP-2.7
 OLDAP-2.8
 OML
 OPL-1.0
+OPUBL-1.0
 OSET-PL-2.1
 OSL-1.0
 OSL-1.1
@@ -305,9 +386,13 @@ PHP-3.0
 PHP-3.01
 PSF-2.0
 Parity-6.0.0
+Parity-7.0.0
 Plexus
+PolyForm-Noncommercial-1.0.0
+PolyForm-Small-Business-1.0.0
 PostgreSQL
 Python-2.0
+Python-2.0.1
 QPL-1.0
 Qhull
 RHeCos-1.1
@@ -336,6 +421,7 @@ SSH-short
 SSPL-1.0
 SWL
 Saxpath
+SchemeReport
 Sendmail
 Sendmail-8.23
 SimPL-2.0
@@ -369,6 +455,7 @@ WTFPL
 Watcom-1.0
 Wsuipa
 X11
+X11-distribute-modifications-variant
 XFree86-1.1
 XSkat
 Xerox
@@ -386,6 +473,7 @@ Zlib
 blessing
 bzip2-1.0.5
 bzip2-1.0.6
+checkmk
 copyleft-next-0.3.0
 copyleft-next-0.3.1
 curl
@@ -400,7 +488,10 @@ iMatix
 libpng-2.0
 libselinux-1.0
 libtiff
+libutil-David-Nugent
+mpi-permissive
 mpich2
+mplus
 psfrag
 psutils
 wxWindows

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:
@@ -211,7 +215,7 @@ def main():
         with open (filename, "r") as myfile:
             lines = myfile.readlines()
     else:
-        output = subprocess.check_output("git format-patch -1 --stdout", shell=True).decode("latin-1")
+        output = subprocess.check_output("git format-patch -1 --stdout", shell=True).decode("utf-8", errors="replace")
         lines = output.split("\n")
         
     parse_patch(lines)

pkg-mapping

@@ -1,2 +1,3 @@
+compat-libsigc++-soname20	compat-libsigc-plus-plus-soname20
 gtk+	gtk-plus
 libsigc++	libsigc-plus-plus

vendor.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+import requests
+from git import Repo
+
+
+def vendor_check():
+    if not os.path.isfile('options.conf'):
+        return False
+    config = configparser.ConfigParser(interpolation=None)
+    config.read('options.conf')
+    if 'autospec' not in config.sections():
+        return False
+    if vendor := config['autospec'].get('cargo_vendor'):
+        if vendor == "true":
+            return 'cargo'
+
+    return False
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
+    parser.add_argument('url')
+    parser.add_argument('name')
+    parser.add_argument('git')
+    return parser.parse_args()
+
+
+def setup_content(url):
+    tdir = tempfile.mkdtemp()
+    outfile = os.path.join(tdir, os.path.basename(url))
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()
+    with open(outfile, 'wb') as cfile:
+        cfile.write(response.content)
+
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
+    os.remove(outfile)
+    return tdir
+
+
+def setup_cargo_vendor(path):
+    cargo_paths = []
+    for dirpath, _, files in os.walk(path):
+        for fname in files:
+            if fname == "Cargo.toml":
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
+
+
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
+    git_uri = os.path.join(git, name)
+    vendor_path = os.path.join(tmpdir, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    vendor_git = os.path.join(vendor_path, '.git')
+    if not os.path.isdir(vendor_git):
+        # initialize a git repo
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
+        subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    shutil.rmtree(vendor_path)
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
+                                   check=True, stdout=subprocess.PIPE,
+                                   universal_newlines=True).stdout
+    with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
+        gafile.write("* text=false\n")
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    repo = Repo(vendor_path)
+    if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
+        # Always use the newest tag as sometimes a new tag will
+        # be created but the package won't be updated to use it
+        # for a different failure reason.
+        tag = sorted(repo.tags, key=lambda x: x.name, reverse=True)[0]
+        return tag, cargo_vendors
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    gmt = time.gmtime()
+    tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
+    subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
+                   check=True, stdout=subprocess.DEVNULL)
+    subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    time.sleep(30)
+    return tag, cargo_vendors
+
+
+def update_cargo_sources(name, tag, cargo_vendors):
+    makefile = []
+    options = []
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
+                                 'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
+    with open('Makefile', encoding='utf8') as mfile:
+        for line in mfile.readlines():
+            if line.startswith('ARCHIVES'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
+                makefile.append(new_archives)
+            else:
+                makefile.append(line)
+    with open('Makefile', 'w', encoding='utf8') as mfile:
+        mfile.writelines(makefile)
+
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+    with open('cargo_vendors', 'w', encoding='utf8') as cfile:
+        cfile.write(cargo_vendors)
+
+
+def main():
+    args = get_args()
+
+    vtype = vendor_check()
+    if not vtype:
+        print(args.archives)
+        return
+
+    tdir = setup_content(args.url)
+    if vtype == 'cargo':
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
+            print(args.archives)
+        else:
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
+            update_cargo_sources(args.name, tag, cargo_vendors)
+    shutil.rmtree(tdir)
+
+
+if __name__ == '__main__':
+    main()