Update packages file for version 34280

Signed-off-by: clrbuilder <clrbuilder@intel.com>
Update packages file for version 34270
2026-09-06 13:41:33 +00:00 · 2021-02-16 00:08:08 -08:00 · 2021-02-11 00:07:31 -08:00 · 2021-02-09 00:08:27 -08:00 · 2021-02-04 00:07:32 -08:00 · 2021-02-02 00:09:39 -08:00
24 changed files with 1784 additions and 933 deletions
@@ -9,9 +9,11 @@ include $(TOPLVL)/projects/common/Makefile.shared
 SHELL  = /bin/bash
 CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 SPECFILE = $(addsuffix .spec, $(PKG_NAME))
-SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || rpmspec -D '_vendor clr' --srpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' $(SPECFILE))
+SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || $(call queryspec,%{NVR}\n,$(SPECFILE)))
 SRPMFILE = results/$(SRPMVERS).src.rpm
 LATEST_RPMS = $(wildcard rpms/*.rpm)
@@ -21,7 +23,8 @@ RPMS ?= $(LATEST_RPMS)
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
 # MOCK_CONFIG_VAL is set in Makefile.shared
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_OPTS)
+MOCK_SMP_FLAGS=$(patsubst -j%, -D_smp_mflags\ -j%,$(filter -j%, $(MAKEFLAGS)))
 MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_SMP_FLAGS) $(MOCK_OPTS)
 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
@@ -45,17 +48,10 @@ upstream:
 #help spdxcheck: Checks that the package license is a valid SPDX license,
 #help or that it is in the exceptions list.
 spdxcheck: $(SPECFILE)
-	@for LICENSE in `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{LICENSE}\n' $(SPECFILE)`; do \
+	@for LICENSE in `$(call queryspec,%{LICENSE}\n,$(SPECFILE))`; do \
 		if ! grep -qx "$$LICENSE" $(TOPLVL)/projects/common/licenses-{spdx,exceptions,extra}; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
 	done; test -z "$$FAIL"
 #help cvecheck: Checks
 #help for common vulnerabilities and exposures in your package.
 cvecheck:
 	# https://github.com/ikeydoherty/cve-check-tool
 	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
 	cve-check-tool -M $(TOPLVL)/projects/common/mapping $(SPECFILE)
 #help clean: Cleans up the src.rpm file. but not the rpm files or log files.
 clean:
 	rm -f $(SRPMFILE)
@@ -143,6 +139,10 @@ pullrebase:
 			echo "No commits exist in remote. Continuing."; \
 			exit 0; \
 		fi; \
 		if [ "$$(git rev-list FETCH_HEAD | head -n 1)" = "$$(git rev-list HEAD | head -n 1)" ]; then \
 			echo "Current branch up-to-date. Continuing."; \
 			exit 0; \
 		fi; \
 		echo "Rebasing to origin/master"; \
 		if git status --porcelain | grep -q '^.[^?]'; \
 		then \
@@ -175,9 +175,11 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		--name $(PKG_NAME) \
 		--archives $(ARCHIVES) \
 		--mock-config $(MOCK_CONFIG_VAL) \
-		--mock-opts "$(MOCK_OPTS)" \
+		--mock-opts="$(MOCK_SMP_FLAGS) $(MOCK_OPTS)" \
 		$${SETVERSION:+ --version $${SETVERSION}} \
-		${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+		$${NON_INTERACTIVE:+ --non_interactive} \
 		$${SKIP_GIT:+ --skip-git} \
 		$${CLEANUP:+ -C} \
 		$(firstword $(NEWURL) $(URL));
 	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@$(MAKE) spdxcheck
@@ -187,6 +189,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		git commit --amend --no-edit Change* ; \
 	fi
 	@$(MAKE) -s for-review.txt
 	diffstat for-review.txt
 	@printf "\n**\n"
 	@printf "** NOTICE: A patch with changes is available in the file for-review.txt\n"
 	@printf "** To recreate (e.g., after git commit --amend), run make for-review.txt\n"
@@ -201,12 +204,12 @@ for-review.txt:
 #help autospecnogit: Runs autospec, but does not create a commit
 autospecnogit:
-	$(MAKE) SKIP_GIT=--skip-git autospec
+	$(MAKE) autospec SKIP_GIT=1
 #help autospecnostate: Runs autospec, but cleans up mock chroots
 #help and disables interactive mode.
 autospecnostate:
-	$(MAKE) CLEANUP=-C NON_INTERACTIVE=--non_interactive autospec
+	$(MAKE) autospec CLEANUP=1 NON_INTERACTIVE=1
 scanlicense:
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --config $(AUTOSPEC_CONF) --license-only $(firstword $(NEWURL) $(URL)) --name $(PKG_NAME)
@@ -215,13 +218,13 @@ scanlicense:
 bump:
 	git pull --rebase
 	$(MAKE) bumpnogit
-	git add *.spec release
+	git add $(SPECFILE) release
-	git commit -a -m "version bump from $(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)) to `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)`"
+	git commit -a -m "version bump from $(shell $(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))) to `$(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))`"
 #help bumpnogit: Increments the release file by one and does not commit changes.
 bumpnogit:
 	@$(MAKE) spdxcheck
-	oldrel=$(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{RELEASE}\n' $(SPECFILE)); \
+	oldrel=$(shell $(call queryspec,%{RELEASE}\n,$(SPECFILE))); \
 	newrel=$$(($$oldrel + 1)); \
 	sed -i "s/^\(Release[[:blank:]]*:[[:blank:]]*\)$$oldrel$$/\1$$newrel/" $(SPECFILE); \
 	echo "$$newrel" > release;
@@ -231,7 +234,7 @@ $(__missingsources): sources
 # First argument is a Clear Linux build number (e.g. 22000)
 define fetch-from-srpm
 	tmp=$$(mktemp -d -p "."); \
-	srpm=$$(rpmspec --srpm -q --qf "%{NVR}.src.rpm\n" $(SPECFILE)); \
+	srpm=$$($(call queryspec,%{NVR}.src.rpm\n,$(SPECFILE))); \
 	if cd "$$tmp" && ! curl -s -S -f -L -O $(MIRROR_CURL_OPTS) $(DOWNLOAD_MIRROR)/releases/$(1)/clear/source/SRPMS/$$srpm; then \
 		echo "Failed to download $$srpm from Clear Linux release $(1)." >&2; \
 		cd .. && rm -rf "$$tmp"; \
@@ -239,7 +242,7 @@ define fetch-from-srpm
 	fi; \
 	rpm2cpio $$srpm | cpio --quiet -id; \
 	cd ..; \
-	srclist=$$(rpmspec --srpm -q --qf "[%{SOURCE}\n]" $(SPECFILE)); \
+	srclist=$$($(call queryspec,[%{SOURCE}\n],$(SPECFILE))); \
 	for s in $$srclist; do \
 		if ! mv --no-clobber "$$tmp"/$$s .; then \
 			echo "Missing source file \"$$s\" in $${srpm}." >&2; \
@@ -278,7 +281,8 @@ else
 sources:
 	@$(MAKE) generateupstream; \
 	[ $$? -eq 0 ] && exit 0; \
-	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	nvr="$$($(call queryspec,%{NAME}\t%{VERSION}\t%{RELEASE}\n,$(SPECFILE)))"; \
 	git -C $(TOPLVL)/projects/common fetch --tags >/dev/null 2>&1; \
 	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
 	for b in $$latest_builds; do \
 		url="$(DOWNLOAD_MIRROR)/releases/$$b/clear/source/package-sources"; \
@@ -303,19 +307,24 @@ prekoji-checks:
 koji: prekoji-checks kojidef
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
-	@if ! git branch | grep -q -E '^\* master'; then \
+	@if [ "$(CURRENT_BRANCH)" != "master" ] && [ "$(CURRENT_BRANCH)" != "main" ]; then \
-		echo "Error: Must be on the master branch to submit to koji" >&2; \
+		echo "Error: Must be on the \"master\" or \"main\" branch to submit to koji" >&2; \
 		exit 1; \
 	fi; \
-	if ! git diff --quiet HEAD ${SPECFILE}; then \
+	if ! git diff --quiet HEAD -- ${SPECFILE}; then \
 		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
 		exit 1; \
 	fi; \
-	if git rev-parse --verify --quiet origin/master > /dev/null; then \
+	git fetch; \
 	if git rev-parse --verify --quiet origin/master >/dev/null; then \
 		git pull --rebase; \
 	fi; \
 	if git ls-remote --tags --exit-code origin $(SRPMVERS) >/dev/null; then \
 		echo "Error: remote tag $(SRPMVERS) already exists" >&2; \
 		exit 1; \
 	fi
 	git tag $(SRPMVERS)
-	git push origin master refs/tags/$(SRPMVERS)
+	git push origin $(CURRENT_BRANCH):master refs/tags/$(SRPMVERS)
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
 #help rekoji: In case a previous 'make koji' failed, trigger a rebuild for the
@@ -343,11 +352,6 @@ rekoji-nowait:
 logcheck:
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 $(TOPLVL)/repo:
 	@echo "Creating local RPM repository $(TOPLVL)/repo"
 	mkdir $(TOPLVL)/repo
 	$(MAKE) localrepocreate
 #help repoadd: Adds locally-built RPMs for this package to the local RPM
 #help repository. If this repository does not yet exist, it is created and
 #help enabled.
@@ -355,39 +359,51 @@ repoadd: $(TOPLVL)/repo
 	@if [ -z "${LATEST_RPMS}" ]; then \
 		echo "No rpms found in rpms/ directory."; \
 		exit 1; \
-	fi; \
+	fi
-	if [ -f .repo-index ]; then \
+	$(MAKE) -s repodel NO_CREATEREPO=1
-		echo "Cleaning old rpms:"; \
+	@echo "Adding new rpms:"; \
-		for r in `cat .repo-index`; do \
+	for rpm in ${LATEST_RPMS}; do \
-			echo "-$$r"; \
+		echo "+$${rpm#rpms/}"; \
-			rm -f $(TOPLVL)/repo/$$r; \
+		echo "$${rpm#rpms/}" >> .repo-index; \
-		done; \
+	done; \
-		rm .repo-index; \
+	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
-	fi; \
+	$(MAKE) -s localrepocreate
-	for r in ${LATEST_RPMS}; do \
+	$(MAKE) -s localrepoenable
-		echo "+$$r"; \
+
-		echo $$(basename $$r) >> .repo-index; \
+#help repostage: Stages locally-built RPMs for this package to the local RPM
-		ln -f $$r $(TOPLVL)/repo/; \
+#help directory. If this directory does not yet exist, it is created and
-	done
+#help enabled.
-	$(MAKE) localrepocreate
+repostage: $(TOPLVL)/repo
-	$(call localrepoenable,${PM_CONF},${MOCK_CONF})
+	@if [ -z "${LATEST_RPMS}" ]; then \
 		echo "No rpms found in rpms/ directory."; \
 		exit 1; \
 	fi
 	$(MAKE) -s repodel NO_CREATEREPO=1
 	@for r in ${LATEST_RPMS}; do \
 		echo "+$${rpm#rpms/}"; \
 		echo "$${rpm#rpms/}" >> .repo-index; \
 	done; \
 	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
-repodel:
+repodel: $(TOPLVL)/repo
 	@if [ -f .repo-index ]; then \
 		echo "Cleaning old rpms:"; \
-		for r in `cat .repo-index`; do \
+		mapfile -t rpms < <(sed 's|.*|$</&|' .repo-index); \
-			echo "-$$r"; \
+		for rpm in "$${rpms[@]}"; do \
-			rm -f $(TOPLVL)/repo/$$r; \
+			echo "-$${rpm#$</}"; \
-		done; \
+		done < .repo-index; \
 		flock $</repo.lock rm -f "$${rpms[@]}"; \
 		rm .repo-index; \
-		$(MAKE) localrepocreate; \
+		if [ -z "$$NO_CREATEREPO" ]; then \
 			$(MAKE) -s localrepocreate; \
 		fi; \
 	fi
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
-repoenable: $(TOPLVL)/repo ; $(call localrepoenable,${PM_CONF},${MOCK_CONF})
+repoenable: localrepoenable ;
 #help repodisable: Disables the local RPM repository.
 repodisable: localrepodisable ;
@@ -484,7 +500,7 @@ generateupstream:
 				exit 1; \
 			fi; \
 		fi; \
-		echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
+		echo $$(sha1sum -- $$filename | cut -d\  -f1)/$$filename >> upstream; \
 	done
 	@rm -f upstream.bak
 	@cat upstream
@@ -502,6 +518,9 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 # Define LTS-specific targets in a separate makefile
 -include $(TOPLVL)/projects/common/Makefile.common.lts
 # Define site local common targets in a separate makefile
 -include $(TOPLVL)/projects/common/Makefile.common.site_local
@@ -0,0 +1,90 @@
 #-*-makefile-*-
 LTSUTILS = python $(TOPLVL)/projects/common/lts/main.py $(PKG_NAME)
 #help lts-show: Display a summary of active LTS branches.
 lts-show:
 	@$(LTSUTILS) sanity-check
 	@while read b; do \
 		if git show-ref $$b > /dev/null; then \
 			echo $$b $$(git log --oneline -1 $$b); \
 		else \
 			echo $$b Not found; \
 		fi; \
 	done < $(TOPLVL)/projects/common/lts/active-branches
 #help lts-backport: Fast-forward the previous active branch to the current
 #help branch.
 lts-backport:
 	@$(LTSUTILS) sanity-check
 	@newer=$$(git symbolic-ref HEAD); \
 	newer=$${newer#refs/heads/}; \
 	if ! $(LTSUTILS) prev-branch --checkout; then \
 		echo Could not check out previous active branch.; \
 		exit 0; \
 	fi; \
 	if $(LTSUTILS) is-same-version $$newer; then \
 		$(LTSUTILS) fast-forward $$newer; \
 	else \
 		echo Most likely a patch needs to be manually re-applied for this version.; \
 		echo Alternatively, use \"git merge --ff-only $$newer\" to upgrade the package version.; \
 	fi
 #help lts-build: Build RPM in Koji, or reuse existing build from older LTS
 #help branch.
 lts-build:
 	@$(LTSUTILS) sanity-check
 	@tag=$$(git tag --points-at); \
 	if [[ -z $$tag ]]; then \
 		echo This commit does not have a tag.; \
 		echo Proceeding to run \"make lts-koji\".; \
 		$(LTSUTILS) prompt || exit; \
 		$(MAKE) lts-koji; \
 		exit; \
 	fi; \
 	older=$$($(LTSUTILS) prev-branch); \
 	if [[ -z $$older ]]; then \
 		echo Could not determine previous active branch.; \
 		exit 1; \
 	fi; \
 	current=$$($(LTSUTILS) current-branch); \
 	if [[ $$(git rev-parse $$older) != $$(git rev-parse $$current) ]]; then \
 		echo Previous branch and current branch have different commits.; \
 		for b in $$older $$current; do echo $$b $$(git log --oneline -1 $$b); done; \
 		echo Proceeding to run \"make lts-koji\".; \
 		$(LTSUTILS) prompt || exit; \
 		$(MAKE) lts-koji; \
 		exit; \
 	fi; \
 	if $(LTSUTILS) can-reuse-binary $$older; then \
 		echo Reusing binary from $$older.; \
 		$(KOJI_CMD) tag-build $$current $(SRPMVERS); \
 	else \
 		echo Could not reuse binary from $$older.; \
 		echo Proceeding to run \"make bump lts-koji\".; \
 		$(LTSUTILS) prompt || exit; \
 		$(MAKE) bump lts-koji; \
 	fi
 	@$(LTSUTILS) next-branch --checkout
 #help lts-koji: Same as "koji" but for working on LTS branches. Should not
 #help be called directly, use lts-build instead.
 lts-koji: prekoji-checks kojidef
 	@$(LTSUTILS) sanity-check
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
 	@current=$$($(LTSUTILS) current-branch); \
 	if ! grep $$current $(TOPLVL)/projects/common/lts/active-branches; then \
 		echo "Error: Must be on an active branch to submit to koji" >&2; \
 		exit 1; \
 	fi; \
 	if ! git diff --quiet HEAD ${SPECFILE}; then \
 		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
 		exit 1; \
 	fi; \
 	if git rev-parse --verify --quiet origin/$$current > /dev/null; then \
 		git pull --rebase; \
 	fi
 	git tag $(SRPMVERS)-lts
 	git push origin $$($(LTSUTILS) current-branch) refs/tags/$(SRPMVERS)-lts
 	$(KOJI_CMD) build $$KOJI_NOWAIT $$($(LTSUTILS) current-branch) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)-lts
@@ -47,6 +47,14 @@ define subjectprefix
 	git config format.subjectPrefix "PATCH $(1)"
 endef
 # Query the spec file (or spec files) listed in argument 2 for the field(s)
 # specified in argument 1. The fields are derived from the source rpm(s)
 # generated by the specs. Queried fields that are declared within the spec file
 # must be declared above the first %files section.
 define queryspec
 rpmspec --define='_vendor clr' --srpm --query --queryformat='$(1)' <(sed -n '1,/%files/p' $(2))
 endef
 # If GITOLITE_BASE_URL is defined, sets a repo's push URL for that gitolite
 # instance. Accepts one argument: the path to the repo on gitolite. If
 # GITOLITE_BASE_URL is not defined, no push URL is set -- the shell's null
@@ -62,33 +70,95 @@ define remotepkgname
 $(if $(USE_PACKAGE_MAPPING),$$(awk -v P="$(1)" '$$1 == P { res=$$2 } END { print res ? res : P }' $(TOPLVL)/projects/common/pkg-mapping),$(1))
 endef
 # Clone repo from the configured gitolite instance, but only if the repo exists
 # on that instance and matches the pattern "packages/..*".
 define try-clone-gitolite
 rpkg=$(call remotepkgname,$(1)); \
 if ssh $(GITOLITE_BASE_URL) info "packages/$$rpkg" | grep --quiet "packages/$$rpkg$$"; then \
 	git clone $(PKG_BASE_URL)/$$rpkg packages/$(1); \
 	cd packages/$(1); \
 	$(call gitoliteurl,packages/$(1)); \
 	$(call subjectprefix,$(1)); \
 fi
 endef
 # Clone repo from the configured package repo hosting location, as defined by
 # the PKG_BASE_URL variable, but only if the repo exists.
 define try-clone-other
 rpkg=$(call remotepkgname,$(1)); \
 if git ls-remote $(PKG_BASE_URL)/$$rpkg &> /dev/null; then \
 	git clone $(PKG_BASE_URL)/$$rpkg packages/$(1); \
 	cd packages/$(1); \
 	$(call subjectprefix,$(1)); \
 fi
 endef
 # First, try running `make clone_PKG`, where PKG is the first argument. If that
 # fails, proceed with fallback cloning strategies.
 define clone-if-available
 $(MAKE) clone_$(1) || { \
 	$(if $(GITOLITE_BASE_URL),$(call try-clone-gitolite,$(1)),$(call try-clone-other,$(1))); \
 }
 endef
 $(TOPLVL)/repo:
 	@echo "Creating local RPM repository $(TOPLVL)/repo"
 	mkdir $(TOPLVL)/repo
 # Enables the local RPM repo by installing package manager and Mock configs.
 # First argument is the path to the package manager config, and the second
 # argument is the path to the mock config. Each of these config files is
 # augmented for local repo support.
-define localrepoenable
+localrepoenable: $(TOPLVL)/repo
-	@cp $(1) $(TOPLVL)/repo/yum.conf
+	@if grep -qx '\[localrepo\]' ${PM_CONF}; then \
-	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/yum.conf
+		echo "[ERROR] ${PM_CONF} contains section named [localrepo]."; \
-	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/yum.conf
+		echo "[ERROR] Remove that section and try again."; \
-	@cp $(2) $(TOPLVL)/repo/clear.cfg
+		exit 1; \
-	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/clear.cfg
+	fi >&2
-	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/clear.cfg
+	@if grep -qx '\[localrepo\]' ${MOCK_CONF}; then \
-	@sed -i '/^"""/d; $$a"""' $(TOPLVL)/repo/clear.cfg
+		echo "[ERROR] ${MOCK_CONF} contains section named [localrepo]."; \
-endef
+		echo "[ERROR] Remove that section and try again."; \
 		exit 1; \
 	fi >&2
 	@last="$$(tail -n 1 ${MOCK_CONF})"; \
 	if [[ "$$last" != "\"\"\"" ]]; then \
 		echo "[ERROR] Last line of ${MOCK_CONF} is malformed; expected: \"\"\"; actual: $$last"; \
 		echo "[ERROR] Fix that line and try again."; \
 		exit 1; \
 	fi >&2
 	@pm_new=$$(mktemp -p $< yum.conf.XXXXXXX); \
 	pm_orig=$</yum.conf; \
 	mock_new=$$(mktemp -p $< clear.cfg.XXXXXXX); \
 	mock_orig=$</clear.cfg; \
 	cp ${PM_CONF} $$pm_new; \
 	printf "\n\n[localrepo]\nname=localrepo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $$pm_new; \
 	printf "baseurl=file://$$(realpath $<)/\n\n" >> $$pm_new; \
 	if ! cmp -s $$pm_new $$pm_orig; then \
 		flock $</repo.lock mv $$pm_new $$pm_orig; \
 	fi; \
 	rm -f $$pm_new; \
 	cp ${MOCK_CONF} $$mock_new; \
 	sed -i '$$d' $$mock_new; \
 	printf "\n\n[localrepo]\nname=localrepo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $$mock_new; \
 	printf "baseurl=file://$$(realpath $<)/\n\n" >> $$mock_new; \
 	sed -i '$$a"""' $$mock_new; \
 	if ! cmp -s $$mock_new $$mock_orig; then \
 		flock $</repo.lock mv $$mock_new $$mock_orig; \
 	fi; \
 	rm -f $$mock_new
 # Disables the local RPM repo by removing the installed package manager and
 # Mock configuration files.
-localrepodisable:
+localrepodisable: $(TOPLVL)/repo
-	@rm -f $(TOPLVL)/repo/yum.conf
+	flock $</repo.lock rm -f $</yum.conf $</clear.cfg
 	@rm -f $(TOPLVL)/repo/clear.cfg
-# Runs createrepo_c (or createrepo) for the local RPM repo
+# Runs createrepo_c for the local RPM repo
-localrepocreate:
+localrepocreate: $(TOPLVL)/repo
-	( cd $(TOPLVL)/repo ; if type createrepo_c 1>/dev/null 2>&1; then createrepo_c .; else createrepo .; fi );
+	cd $< && flock repo.lock createrepo_c .
 # Remove all RPMs from the local RPM repo
-localrepoclean:
+localrepoclean: $(TOPLVL)/repo
-	@find $(TOPLVL)/repo -maxdepth 1 -name '*.rpm' -delete
+	flock $</repo.lock find $< -maxdepth 1 -name '*.rpm' -delete
 # Summarizes the local RPM repo status
 localrepostatus:
@@ -34,7 +34,7 @@ $(proj_PKGS):
 	git clone $(PRJ_BASE_URL)/$$proj projects/$$proj; \
 	cd projects/$$proj; \
 	if [ "$$proj" = "autospec" ] ; then \
-		git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+		git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 	else \
 		$(call gitoliteurl,projects/$$proj); \
 	fi; \
@@ -56,12 +56,6 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	$(call subjectprefix,$$pkg); \
 	)
 #help cvecheck: Checks for common vulnerabilities and exposures in your code.
 cvecheck:
 	# https://github.com/ikeydoherty/cve-check-tool
 	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
 	cve-check-tool -n -M $(TOPLVL)/projects/common/mapping $(PACKAGES_FILE)
 #help pull: Performs a git pull --rebase for each package repo, avoiding the creation
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
@@ -83,7 +77,7 @@ ${PULL_PKGS}:
 			git --no-pager log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit $$O..$$N; echo; \
 		fi ; \
 		if [ "$$p" = "projects/autospec" ] ; then \
-			git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+			git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 		else \
 			$(call gitoliteurl,$$p); \
 		fi ; \
@@ -172,16 +166,25 @@ diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 		(cd $$p ; git status | grep -q 'nothing to commit, working [^ ]* clean' || (echo "Uncommitted changes in: $$p"; git diff |diffstat -p1)) ;\
 	done
 #help log: Run git log origin/master..HEAD for all package repos, thus
 #help displaying any committed but unpushed changes.
 log: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
 		( cd $$p; if [ "$$(git rev-parse --verify HEAD)" != "$$(git rev-parse --verify origin/master)" ] ; then \
 		echo $$p ; git --no-pager log --pretty="format:%s" origin/master..HEAD ; echo ; echo;\
 		fi ; ) \
 	done
 #help versions: Displays the version of each package in the ./packages tree.
 versions: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
-		(cd $$p; if ls *.spec &> /dev/null; then rpmspec -q --queryformat '%{NAME}-%{VERSION}\n' *.spec | head -n1; fi) ;\
+		(cd $$p; if ls *.spec &> /dev/null; then $(call queryspec,%{NAME}-%{VERSION}\n,*.spec) | head -n1; fi) ;\
 	done
 #help releases: Like 'make versions', but also displays the release number.
 releases: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
-		(cd $$p; if ls *.spec &> /dev/null; then rpmspec -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' *.spec | head -n1; fi) ;\
+		(cd $$p; if ls *.spec &> /dev/null; then $(call queryspec,%{NVR}\n,*.spec) | head -n1; fi) ;\
 	done
 #help provides:     Is used to find out which RPM package provides some file.
@@ -202,7 +205,7 @@ autospecnew: preautospecnew-checks localreponotice
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
-	-$(MAKE) clone_$(NAME)
+	$(call clone-if-available,$(NAME))
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
@@ -221,7 +224,7 @@ autospecnew: preautospecnew-checks localreponotice
 			--name $(NAME) \
 			--archives $(ARCHIVES) \
 			--mock-config $(MOCK_CONFIG_VAL) \
-			--mock-opts "$(MOCK_OPTS)" \
+			--mock-opts="$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
 			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 			$(URL); \
@@ -233,20 +236,63 @@ autospecnew: preautospecnew-checks localreponotice
 			printf "** NOTICE: A patch with changes is available in the file $(TOPLVL)/packages/$(NAME)/for-review.txt\n"; \
 			printf "** Please look through this file and if you are unsure, please submit for code review with git send-email\n"; \
 			printf "**\n\n"; \
 		else \
 			echo "Autospec of $(NAME) failed."; \
 			exit 1; \
 		fi; \
 	else \
 		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
 		exit 1; \
 	fi
-$(TOPLVL)/repo:
+#help autospecnewgo: Creates a new autospec package with (dependencies)
-	@echo "Creating local RPM repository $(TOPLVL)/repo"
+#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
-	mkdir $(TOPLVL)/repo
+#help autospec will be created in the help process.
-	$(MAKE) localrepocreate
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
 #help to autospec.
 #help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
 autospecnewgo: preautospecnew-checks localreponotice
 	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
 	-$(MAKE) clone_$(NAME)
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
 		( \
 			cd $(TOPLVL)/packages/$(NAME); \
 			git init; \
 			git remote add origin $(PKG_BASE_URL)/$(NAME); \
 			$(call gitoliteurl,packages/$(NAME)); \
 			$(call subjectprefix,$(NAME)); \
 		); \
 		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
 		python3 $(TOPLVL)/projects/common/go-builder.py \
 			--target packages/$(NAME) \
 			--integrity \
 			--config "$(AUTOSPEC_CONF)" \
 			--name $(NAME) \
 			--archives $(ARCHIVES) \
 			--mock-config $(MOCK_CONFIG_VAL) \
 			--mock-opts "$(MOCK_OPTS)" \
 			$(URL); \
 		if [ $$? -eq 0 ]; then \
 			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
 			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
 			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
 			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
 		else \
 			exit 1; \
 		fi; \
 	else \
 		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
 		exit 1; \
 	fi
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
-repoenable: $(TOPLVL)/repo ; $(call localrepoenable,${PM_CONF},${MOCK_CONF})
+repoenable: localrepoenable ;
 #help repodisable: Disables the local RPM repository.
 repodisable: localrepodisable ;
@@ -0,0 +1,40 @@
 ## Advanced topics
 ### Manual setup
 If you did not run the user-setup script (see "Automated setup" section in the
 [main README](README.md), you will want to set up the developer tooling
 workspace manually. This section provides general documentation for the manual
 setup process, and it is not meant to be exhaustive.
 On your Clear Linux system, create a workspace for Clear Linux development
 work:
 ```
 $ mkdir clearlinux
 ```
 Clone this repo into a `projects` directory within the workspace:
 ```
 $ cd clearlinux
 $ mkdir projects
 $ git clone https://github.com/clearlinux/common projects/common
 ```
 Create the toplevel tooling Makefile:
 ```
 $ ln -s projects/common/Makefile.toplevel Makefile
 ```
 Clone all Clear Linux package and project repositories:
 ```
 $ make clone
 ```
 Note: You can clone the repos in parallel by using make's `-j` option.
 At this point, the `packages` directory will contain all Clear Linux package
 repos, and `projects` will contain common, clr-bundles, and autospec repos.
@@ -152,39 +152,4 @@ If `Makefile.config.site_local` doesn't exist already, create it.
 ### Manual setup
-If you did not run the user-setup script (see "Automated setup" section above),
+See the [Manual setup](README-advanced.md#manual-setup) documentation.
 you will want to set up the developer tooling workspace manually. This section
 provides general documentation for the manual setup process, and it is not
 meant to be exhaustive.
 On your Clear Linux system, create a workspace for Clear Linux development
 work:
 ```
 $ mkdir clearlinux
 ```
 Clone this repo into a `projects` directory within the workspace:
 ```
 $ cd clearlinux
 $ mkdir projects
 $ git clone https://github.com/clearlinux/common projects/common
 ```
 Create the toplevel tooling Makefile:
 ```
 $ ln -s projects/common/Makefile.toplevel Makefile
 ```
 Clone all Clear Linux package and project repositories:
 ```
 $ make clone
 ```
 Note: You can clone the repos in parallel by using make's `-j` option.
 At this point, the `packages` directory will contain all Clear Linux package
 repos, and `projects` will contain common, clr-bundles, and autospec repos.
@@ -0,0 +1,285 @@
 #!/usr/bin/env python3
 #
 # gowrap.py - part of autospec
 # Copyright (C) 2019 Intel Corporation
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import argparse
 import os
 import shutil
 import subprocess
 import sys
 import tempfile
 def parse_args():
    """Parse commandline arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("url", help="Location of package archive")
    parser.add_argument("-t", "--target", dest="target", action="store",
                        default=None,
                        help="Target location to create or reuse")
    parser.add_argument("-c", "--config", dest="config", action="store",
                        default="/usr/share/defaults/autospec/autospec.conf",
                        help="Set configuration file to use")
    parser.add_argument("-n", "--name", action="store", dest="name", default="",
                        help="Override the package name")
    parser.add_argument("-i", "--integrity", action="store_true",
                        default=False,
                        help="Search for package signature from source URL and "
                             "attempt to verify package")
    parser.add_argument("-m", "--mock-config", action="store", default="clear",
                        help="Value to pass with Mock's -r option. Defaults to "
                             "\"clear\", meaning that Mock will use "
                             "/etc/mock/clear.cfg.")
    parser.add_argument("-o", "--mock-opts", action="store", default="",
                        help="Arbitrary options to pass down to mock when "
                        "building a package.")
    parser.add_argument('-a', "--archives", action="store",
                        dest="archives", default=[], nargs='*',
                        help="tarball URLs for additional source archives and"
                        " a location for the sources to be extacted to (e.g."
                        " http://example.com/downloads/dependency.tar.gz"
                        " /directory/relative/to/extract/root )")
    return parser.parse_args()
 def missing_dependencies(name):
    """Check if the build failed due to missing dependencies."""
    root_log = os.path.join("packages", name, "results", "root.log")
    if not os.path.exists(root_log):
        return False
    with open(root_log, "r") as lfile:
        for line in lfile.readlines():
            if "No matching package to install" in line:
                return True
    return False
 def already_built(name, version):
    """Check if the package and version are already built."""
    if not os.path.exists(os.path.join("packages", name, "rpms")):
        return False
    vpath = os.path.join("packages", name, "versions")
    if not os.path.exists(vpath):
        return False
    with open(vpath, "r") as vfile:
            if version not in [x.strip() for x in vfile.readlines()]:
                return False
    return True
 def build(path, args, name, url, version):
    """Try and build a package."""
    if version:
        ver_arg = ["-v", version]
    else:
        ver_arg = []
    if already_built(name, version):
        return True
    print(f"Trying to autospec {name} - {version}")
    proc = subprocess.run(["python3",
                           f"{path}/../autospec/autospec/autospec.py", url,
                           "-c", args.config, "-t", f"packages/{name}",
                           "-n", name, "-m", args.mock_config,
                           "-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
    if proc.returncode == 0:
        subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
                       capture_output=True)
    return proc.returncode == 0
 def parse_go_mod(path):
    """Parse go.mod file for build requirements.
    File content looks as follows:
    module example.com/foo/bar
    require (
        github.com/BurntSushi/toml v0.3.1
        git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
        github.com/inconshreveable/mousetrap v1.0.0 // indirect
        "github.com/spf13/cobra" v0.0.3
        github.com/spf13/pflag v1.0.3 // indirect
    )
    Need to handle all require lines including //indirect.
    Skip requires that use .git for now. May need to be handled
    differently.
    """
    reqs = []
    with open(path, "r") as gfile:
        dep_start = False
        for line in gfile.readlines():
            # Ideally the mod file is generated and the format is
            # always correct but add a few defenses just in case
            line = line.strip()
            if line.startswith("//"):
                # Skip comments
                continue
            if dep_start:
                # End of the require section
                if line.startswith(")"):
                    break
                req = line.split()[:2]
                req[0] = req[0].replace('"', '')
                if req[0].endswith(".git"):
                    continue
                reqs.append(req)
                continue
            if line.startswith("require ("):
                dep_start = True
    return reqs
 def get_dependencies(name):
    """Return path to the go.mod file if it exists."""
    command = None
    path = None
    pdir = os.path.join("packages", name)
    reqs = []
    for fname in os.listdir(pdir):
        # Find the archive
        if fname.endswith("zip"):
            command = ["unzip", fname]
        elif ".tar." in fname:
            command = ["tar", "xf", fname]
        else:
            command = None
        if command:
            # Decompress archive
            tdir = tempfile.mkdtemp()
            tfile = os.path.join(tdir, fname)
            shutil.copyfile(os.path.join(pdir, fname), tfile)
            proc = subprocess.run(command, cwd=tdir, capture_output=True)
            if not proc.returncode:
                # inspect contents for a "go.sum" file
                for root, _, files in os.walk(tdir):
                    if "go.mod" in files:
                        path = f"{name}.gomod"
                        shutil.copyfile(os.path.join(root, "go.mod"), path)
                        reqs += parse_go_mod(path)
                        os.unlink(path)
            shutil.rmtree(tdir)
    return reqs
 def encode_requirements(requirement):
    """Encode the project's module and version."""
    # Default dependency base proxy url
    base_url = "https://proxy.golang.org/"
    # Encoding based on
    # https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
    encoded_module = ""
    for char in requirement[0]:
        if char.isupper():
            encoded_module += "!" + char.lower()
        else:
            encoded_module += char
    encoded_version = ""
    for char in requirement[1]:
        if char.isupper():
            encoded_version += "!" + char.lower()
        else:
            encoded_version += char
    return (encoded_module, encoded_version)
 def initialize_package(name, url, version, path, args):
    """Setup package and try to build it."""
    if already_built(name, version):
        return True
    print(f"Trying to autospecnew {name} - {version}")
    proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
    if f"{name} already exists at" in proc.stdout.decode("utf-8"):
        return build(path, args, name, url, version)
    return proc.returncode == 0
 def build_recursive(path, args, name, url, version=None, success=None):
    """Try and recursively build packages."""
    if not success:
        # First package being built, common utilities already run
        # so just use autospec directly
        success = {}
        ret = build(path, args, name, url, version)
    else:
        ret = initialize_package(name, url, version, path, args)
    if ret:
        # Completed so we are done!
        return True
    if not missing_dependencies(name):
        # Failed but not due to missing dependencies, bail
        print(f"Unknown error building {name} - {version}")
        return False
    reqs = get_dependencies(name)
    if not reqs:
        # Didn't try and add anything so don't need to build again
        print(f"Build failed with no missing requirements: {name} - {version}")
        return False
    print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
    for req in reqs:
        ereq = encode_requirements(req)
        rname = "go-" + req[0].replace("/", "-")
        rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
        if (rname, req[1]) in success:
            # Rebuild detected, figure out if it is okay or not
            if success[(rname, req[1])]:
                # Already succeeded building req short circuit success
                continue
            else:
                # Encountered a build loop, bail out as this likely
                # requires manual fixing
                print(f"Detected build loop when building {rname} - {req[1]}")
                return True
        success[(rname, req[1])] = False
        if not build_recursive(path, args, rname, rurl, req[1], success):
            print(f"Failed to build dependency {rname} - {req[1]}")
            return False
        success[(rname, req[1])] = True
        subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
        subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
    # Create the repo with the new packages added
    subprocess.run(["make", "localrepocreate"], capture_output=True)
    # Retry previously failed build assuming dependencies got added
    print(f"Rebuilding after resolving dependencies {name} - {version}")
    ret = build(path, args, name, url, version)
    print("")
    return ret
 def main():
    """Start program execution."""
    args = parse_args()
    path = os.path.dirname(os.path.realpath(__file__))
    return build_recursive(path, args, args.name, args.url)
 if __name__ == '__main__':
    if not main():
        sys.exit(-1)
    print("\n\nBuild completed\n\n")
@@ -1,4 +1,5 @@
 Copyright
 Distributable
 ISSL
 Public-Domain
 sendmail
@@ -16,6 +16,7 @@ AMDPLPA
 AML
 AMPAS
 ANTLR-PD
 ANTLR-PD-fallback
 APAFML
 APL-1.0
 APSL-1.0
@@ -39,6 +40,7 @@ BSD-2-Clause
 BSD-2-Clause-FreeBSD
 BSD-2-Clause-NetBSD
 BSD-2-Clause-Patent
 BSD-2-Clause-Views
 BSD-3-Clause
 BSD-3-Clause-Attribution
 BSD-3-Clause-Clear
@@ -46,22 +48,29 @@ BSD-3-Clause-LBNL
 BSD-3-Clause-No-Nuclear-License
 BSD-3-Clause-No-Nuclear-License-2014
 BSD-3-Clause-No-Nuclear-Warranty
 BSD-3-Clause-Open-MPI
 BSD-4-Clause
 BSD-4-Clause-UC
 BSD-Protection
 BSD-Source-Code
 BSL-1.0
 BUSL-1.1
 Bahyph
 Barr
 Beerware
 BitTorrent-1.0
 BitTorrent-1.1
 BlueOak-1.0.0
 Borceux
 CAL-1.0
 CAL-1.0-Combined-Work-Exception
 CATOSL-1.1
 CC-BY-1.0
 CC-BY-2.0
 CC-BY-2.5
 CC-BY-3.0
 CC-BY-3.0-AT
 CC-BY-3.0-US
 CC-BY-4.0
 CC-BY-NC-1.0
 CC-BY-NC-2.0
@@ -72,6 +81,7 @@ CC-BY-NC-ND-1.0
 CC-BY-NC-ND-2.0
 CC-BY-NC-ND-2.5
 CC-BY-NC-ND-3.0
 CC-BY-NC-ND-3.0-IGO
 CC-BY-NC-ND-4.0
 CC-BY-NC-SA-1.0
 CC-BY-NC-SA-2.0
@@ -85,9 +95,12 @@ CC-BY-ND-3.0
 CC-BY-ND-4.0
 CC-BY-SA-1.0
 CC-BY-SA-2.0
 CC-BY-SA-2.0-UK
 CC-BY-SA-2.5
 CC-BY-SA-3.0
 CC-BY-SA-3.0-AT
 CC-BY-SA-4.0
 CC-PDDC
 CC0-1.0
 CDDL-1.0
 CDDL-1.1
@@ -101,6 +114,9 @@ CECILL-B
 CECILL-C
 CERN-OHL-1.1
 CERN-OHL-1.2
 CERN-OHL-P-2.0
 CERN-OHL-S-2.0
 CERN-OHL-W-2.0
 CNRI-Jython
 CNRI-Python
 CNRI-Python-GPL-Compatible
@@ -122,6 +138,7 @@ ECL-1.0
 ECL-2.0
 EFL-1.0
 EFL-2.0
 EPICS
 EPL-1.0
 EPL-2.0
 EUDatagrid
@@ -139,15 +156,28 @@ Fair
 Frameworx-1.0
 FreeImage
 GFDL-1.1
 GFDL-1.1-invariants-only
 GFDL-1.1-invariants-or-later
 GFDL-1.1-no-invariants-only
 GFDL-1.1-no-invariants-or-later
 GFDL-1.1-only
 GFDL-1.1-or-later
 GFDL-1.2
 GFDL-1.2-invariants-only
 GFDL-1.2-invariants-or-later
 GFDL-1.2-no-invariants-only
 GFDL-1.2-no-invariants-or-later
 GFDL-1.2-only
 GFDL-1.2-or-later
 GFDL-1.3
 GFDL-1.3-invariants-only
 GFDL-1.3-invariants-or-later
 GFDL-1.3-no-invariants-only
 GFDL-1.3-no-invariants-or-later
 GFDL-1.3-only
 GFDL-1.3-or-later
 GL2PS
 GLWTPL
 GPL-1.0
 GPL-1.0+
 GPL-1.0-only
@@ -172,7 +202,9 @@ Glide
 Glulxe
 HPND
 HPND-sell-variant
 HTMLTIDY
 HaskellReport
 Hippocratic-2.1
 IBM-pibs
 ICU
 IJG
@@ -223,6 +255,7 @@ MIT-CMU
 MIT-advertising
 MIT-enna
 MIT-feh
 MIT-open-group
 MITNFA
 MPL-1.0
 MPL-1.1
@@ -234,12 +267,17 @@ MTLL
 MakeIndex
 MirOS
 Motosoto
 MulanPSL-1.0
 MulanPSL-2.0
 Multics
 Mup
 NASA-1.3
 NBPL-1.0
 NCGL-UK-2.0
 NCSA
 NGPL
 NIST-PD
 NIST-PD-fallback
 NLOD-1.0
 NLPL
 NOSL
@@ -248,6 +286,7 @@ NPL-1.1
 NPOSL-3.0
 NRL
 NTP
 NTP-0
 Naumen
 Net-SNMP
 NetCDF
@@ -255,12 +294,19 @@ Newsletr
 Nokia
 Noweb
 Nunit
 O-UDA-1.0
 OCCT-PL
 OCLC-2.0
 ODC-By-1.0
 ODbL-1.0
 OFL-1.0
 OFL-1.0-RFN
 OFL-1.0-no-RFN
 OFL-1.1
 OFL-1.1-RFN
 OFL-1.1-no-RFN
 OGC-1.0
 OGL-Canada-2.0
 OGL-UK-1.0
 OGL-UK-2.0
 OGL-UK-3.0
@@ -293,7 +339,12 @@ OpenSSL
 PDDL-1.0
 PHP-3.0
 PHP-3.01
 PSF-2.0
 Parity-6.0.0
 Parity-7.0.0
 Plexus
 PolyForm-Noncommercial-1.0.0
 PolyForm-Small-Business-1.0.0
 PostgreSQL
 Python-2.0
 QPL-1.0
@@ -311,12 +362,17 @@ SCEA
 SGI-B-1.0
 SGI-B-1.1
 SGI-B-2.0
 SHL-0.5
 SHL-0.51
 SISSL
 SISSL-1.2
 SMLNJ
 SMPPL
 SNIA
 SPL-1.0
 SSH-OpenSSH
 SSH-short
 SSPL-1.0
 SWL
 Saxpath
 Sendmail
@@ -336,6 +392,7 @@ TORQUE-1.1
 TOSL
 TU-Berlin-1.0
 TU-Berlin-2.0
 UCL-1.0
 UPL-1.0
 Unicode-DFS-2015
 Unicode-DFS-2016
@@ -365,6 +422,7 @@ Zend-2.0
 Zimbra-1.3
 Zimbra-1.4
 Zlib
 blessing
 bzip2-1.0.5
 bzip2-1.0.6
 copyleft-next-0.3.0
@@ -374,10 +432,12 @@ diffmark
 dvipdfm
 eCos-2.0
 eGenix
 etalab-2.0
 gSOAP-1.3b
 gnuplot
 iMatix
 libpng-2.0
 libselinux-1.0
 libtiff
 mpich2
 psfrag
@@ -0,0 +1,5 @@
 TEST = test.test_ltsutils
 .PHONY: test
 test:
 	PYTHONPATH=. python -m unittest -v -k .Test $(TEST)
@@ -0,0 +1,19 @@
 # LTS package maintenance utility
 This tooling is designed to automate 2 main tasks that are part of
 the package maintenance workflow of Clear Linux LTS. These tasks are:
 - Back-porting of a patch (e.g. security fix) to older branches.
 - Building RPMs with the intent of sharing binaries of older LTS branches to
  newer branches whenever possible.
 There should be no need to run this tool directly. Instead use the following
 targets defined in Makefile.common.lts:
 - lts-show: Show a summary of active LTS branches
 - lts-backport: Attempt to fast-forward the previous active branch to the current branch
 - lts-build: Build RPM in Koji, or reuse existing build from older branch
 "Active" branches correspond to LTS releases that currently have support.
 They are listed in a flat file "active-branches" in "lts" directory, from
 oldest to newest. New entries are added by Clear Linux LTS developers as
 new releases become available, and entries removed as releases become
 obsolete.
@@ -0,0 +1,52 @@
 import pathlib
 from subprocess import PIPE, CalledProcessError
 from .shell import Shell
 class PackageRepo:
    '''Represents a package repository. Most methods are wrappers of git commands.'''
    class UnknownCurrentBranchException(Exception): pass
    class InvalidBranchException(Exception): pass
    def __init__(self, name, path):
        self.name = name
        self.path = pathlib.Path(path)
        self.sh = Shell(self.path)
    def getNVR(self, commit='HEAD'):
        with self.sh.popen(['git', 'show', '{}:{}.spec'.format(commit, self.name)], stdout=PIPE) as specfile:
            nvr = self.sh.run('rpmspec --srpm -q --queryformat %{NVR} /dev/stdin', stdin=specfile.stdout)
        return tuple(nvr.stdout.strip().rsplit('-', maxsplit=2))
    def checkoutBranch(self, branch, allow_remote=False):
        # allow_remote=True allows checking out a new remote-tracking branch
        if not allow_remote and not self.hasBranch(branch):
            raise self.InvalidBranchException(branch)
        self.sh.run_args(['git', 'checkout', branch], capture_output=False)
    def fastForwardBranch(self, old, new):
        self.checkoutBranch(old)
        if not self.hasBranch(new):
            raise self.InvalidBranchException(new)
        self.sh.run_args(['git', 'merge', '--ff-only', new], capture_output=False)
    def getActiveBranches(self):
        toplvl = pathlib.Path(self.path) / '../..'
        common = toplvl / 'projects/common'
        active_branches = common / 'lts/active-branches'
        with active_branches.open() as f:
            return [line.rstrip() for line in f]
    def getCurrentBranch(self):
        try:
            head = self.sh.run('git symbolic-ref HEAD').stdout.strip()
        except CalledProcessError:
            raise self.UnknownCurrentBranchException
        refs_heads = 'refs/heads/'
        assert head.startswith(refs_heads)
        head = head[len(refs_heads):]
        return head
    def hasBranch(self, branch):
        p = self.sh.run_args(['git', 'rev-parse', 'refs/heads/'+branch], check=False)
        return p.returncode == 0
@@ -0,0 +1,27 @@
 import subprocess, shlex
 class Shell:
    # Default options passed to subprocess.run. May be customized per-instance.
    cwd = None
    check = True
    capture_output = True
    text = True
    def __init__(self, cwd=None):
        if cwd: self.cwd = cwd
    def run_args(self, args, **kwargs):
        kwargs1 = {
                'check': self.check,
                'capture_output': self.capture_output,
                'text': self.text,
                'cwd': self.cwd
                }
        kwargs1.update(kwargs)
        return subprocess.run(args, **kwargs1)
    def run(self, cmd, **kwargs):
        return self.run_args(shlex.split(cmd), **kwargs)
    def popen(self, args, **kwargs):
        return subprocess.Popen(args, cwd=self.cwd, **kwargs)
@@ -0,0 +1,161 @@
 #!/usr/bin/python
 import sys,argparse
 from ltsutils.package_repo import PackageRepo
 def log(msg, **kwargs):
    print(msg, file=sys.stderr)
 def init_parser():
    main = argparse.ArgumentParser()
    main.add_argument('package_name', nargs=1)
    subparsers = main.add_subparsers(dest='command', metavar='command', required=True)
    # Package maintenance commands
    p = subparsers.add_parser('prev-branch',
            help='show previous branch')
    p.add_argument('--checkout', action='store_true', help='checkout the branch')
    p = subparsers.add_parser('next-branch',
            help='show next branch')
    p.add_argument('--checkout', action='store_true', help='checkout the branch')
    p = subparsers.add_parser('current-branch',
            help='show current branch')
    p = subparsers.add_parser('is-same-version',
            help='return true if package version is the same as the given branch')
    p.add_argument('branch', nargs=1)
    p = subparsers.add_parser('fast-forward',
            help='fast-forward current branch to a newer branch')
    p.add_argument('branch', nargs=1)
    # RPM build commands
    p = subparsers.add_parser('can-reuse-binary',
            help='check if binary from another branch can be used in current branch')
    p.add_argument('branch', nargs=1)
    # Other commands
    p = subparsers.add_parser('prompt',
            help='prompt user to continue and return appropriate exit code')
    p = subparsers.add_parser('sanity-check',
            help='run sanity checks for data consistency')
    return main
 def prev_branch(args, repo):
    active_branches = repo.getActiveBranches()
    current = repo.getCurrentBranch()
    i = active_branches.index(current)
    if i == 0:
        log('Already on oldest active branch.')
        return False
    prev = active_branches[i-1]
    print(prev)
    if args.checkout:
        repo.checkoutBranch(prev, allow_remote=True)
 def next_branch(args, repo):
    active_branches = repo.getActiveBranches()
    current = repo.getCurrentBranch()
    i = active_branches.index(current)
    if i == len(active_branches)-1:
        log('Already on newest active branch.')
        return False
    next_ = active_branches[i+1]
    print(next_)
    if args.checkout:
        repo.checkoutBranch(next_, allow_remote=False)
 def current_branch(args, repo):
    current = repo.getCurrentBranch()
    print(current)
 def is_same_version(args, other):
    current = repo.getCurrentBranch()
    other = args.branch[0]
    assert repo.hasBranch(other), 'Branch %s not found' % other
    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (current, other)]
    if v1 != v2:
        log('Current version {} does not match version {} on branch {}'.format(v1, v2, other))
    return v1 == v2
 def fast_forward(args, repo):
    current = repo.getCurrentBranch()
    newer = args.branch[0]
    log('Fast-forwarding {} to {}'.format(current, newer))
    repo.fastForwardBranch(current, newer)
 def sanity_check(args, repo):
    ok = True
    # HEAD must point to a branch
    try:
        current = repo.getCurrentBranch()
    except PackageRepo.UnknownCurrentBranchException:
        log('Unknown current branch. Has a branch been checked out?')
        current = None
        ok = False
    # active-branches file must not be empty
    active_branches = repo.getActiveBranches()
    if not len(active_branches):
        log('No active branches defined. Is active-branches file empty?')
        ok = False
    # current branch must be an active branch
    elif current and current not in active_branches:
        log('%s is not an active LTS branch.' % current)
        ok = False
    return ok
 def can_reuse_binary(args, repo):
    # Just compare versions for now
    # TODO: ABI compatibility testing
    current = repo.getCurrentBranch()
    older = args.branch[0]
    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (older, current)]
    return v1 == v2
 def prompt(args, repo):
    import selectors
    timeout = 60
    while True:
        print('Continue? (y/N): ', end='', flush=True)
        with selectors.DefaultSelector() as sel:
            sel.register(sys.stdin, selectors.EVENT_READ)
            events = sel.select(timeout)
        if not len(events):
            print('Timed out after {}s.'.format(timeout))
            return False
        else:
            s = sys.stdin.readline().rstrip('\n')
            if s in ('Y', 'y', 'N', 'n', ''):
                break
    if s in ('Y', 'y'):
        return True
    else:
        print('Cancelled.')
        return False
 if __name__=='__main__':
    args = init_parser().parse_args()
    repo = PackageRepo(args.package_name[0], '.')
    commands = {
            'prev-branch': prev_branch,
            'next-branch': next_branch,
            'current-branch': current_branch,
            'is-same-version': is_same_version,
            'fast-forward': fast_forward,
            'can-reuse-binary': can_reuse_binary,
            'prompt': prompt,
            'sanity-check': sanity_check,
            }
    ret = commands[args.command](args, repo)
    if ret is not None:
        exit(0 if ret else 1)
@@ -0,0 +1,97 @@
 import unittest
 import os, pathlib, tempfile, logging
 import subprocess, functools
 from ltsutils.package_repo import PackageRepo
 import ltsutils.shell
 run = functools.partial(subprocess.run, check=True)
 class Package:
    def __init__(self, name):
        self.name = name
        self.repo_url = 'https://github.com/clearlinux-pkgs/{}.git'.format(name)
 class LTSUtilsTestCase(unittest.TestCase):
    toplvl = pathlib.Path('../../..')
    packages = toplvl / 'packages'
    def cloneOrExtractRepo(self):
        tmpdir = pathlib.Path('/var/tmp/common-lts-test')
        tmpdir.mkdir(mode=0o700, exist_ok=True)
        tarball = tmpdir / '{}.tar.gz'.format(self.package.name)
        if tarball.exists():
            run(['tar', 'xf', tarball, '-C', self.workdir])
        else:
            run(['git', 'clone', self.package.repo_url, self.workdir])
            run(['tar', 'czf', tarball, '-C', self.workdir, '.'])
    def setUp(self):
        self._tmpdir = tempfile.TemporaryDirectory(prefix='test-{}-'.format(self.package.name), dir=self.packages)
        self.workdir = pathlib.Path(self._tmpdir.name)
        self.cloneOrExtractRepo()
        self.repo = PackageRepo(self.package.name, self.workdir)
        self._sh = ltsutils.shell.Shell(self.workdir)
        self._sh.capture_output = False
    def sh(self, cmd):
        return self._sh.run(cmd)
    def sh_stdout(self, cmd):
        return self._sh.run(cmd, capture_output=True).stdout.rstrip()
    def tearDown(self):
        self._tmpdir.cleanup()
 class PackageRepoTestCase(LTSUtilsTestCase):
    def setUp(self):
        super().setUp()
        self.L1 = self.package.L1
        self.L2 = self.package.L2
        self.sh('git branch L1 %s' % self.L1)
        self.sh('git branch L2 %s' % self.L2)
    def testGetNVR(self):
        raise NotImplementedError
    def testHasBranch(self):
        self.assertTrue(self.repo.hasBranch('L2'))
        self.assertFalse(self.repo.hasBranch('L3'))
    def testCheckoutBranch(self):
        self.repo.checkoutBranch('L2')
        self.assertEqual(self.sh_stdout('git rev-parse HEAD'), self.L2)
        self.assertRaises(PackageRepo.InvalidBranchException, self.repo.checkoutBranch, 'L3')
    def testFastForward(self):
        self.repo.fastForwardBranch('L1', 'L2')
        self.assertEqual(self.sh_stdout('git rev-parse L1'), self.L2)
        self.assertEqual(self.sh_stdout('git rev-parse L2'), self.L2)
    def testGetCurrentBranch(self):
        self.repo.checkoutBranch('L2')
        b = self.repo.getCurrentBranch()
        self.assertEqual(b, 'L2')
        self.sh('git checkout --detach L2')
        self.assertRaises(PackageRepo.UnknownCurrentBranchException, self.repo.getCurrentBranch)
 class TestNano(PackageRepoTestCase):
    package = Package('nano')
    package.L1 = '3dcfa09f5217eedf6ec7539af7e243655d3abdb6' # 3.2-54
    package.L2 = 'b8243dd54e8feb16a11474f848b8735f5591cf12' # 3.2-55
    def testGetNVR(self):
        nvr = self.repo.getNVR(self.L2)
        self.assertEqual(nvr, ('nano', '3.2', '55'))
 class TestMySQL_Python(PackageRepoTestCase):
    package = Package('MySQL-python')
    package.L1 = '386163d8fc9c857c7194c4e958374af4c4f071ed' # 1.2.5-31
    package.L2 = 'f85bc5ec2141384f45f224d4464a0a44a981a4d4' # 1.2.5-33
    def testGetNVR(self):
        nvr = self.repo.getNVR(self.L2)
        self.assertEqual(nvr, ('MySQL-python', '1.2.5', '33'))
@@ -51,4 +51,5 @@ rack = rubygem-rack
 compute = nova
 international_components_for_unicode = icu4c
 node.js = nodejs
-jupyter-core = jupyter_core
+jupyter-core = jupyter_core
 flatten-json = flatten_json
@@ -102,6 +102,22 @@ def zap_entire_file_end(filename):
        if file.endswith(filename):
            files.remove(file)                
 def zap_line_in_file_substring(filename, match):
    global header
    global files
    global files_chunks
    global files_header
    for file in files:
        if file.endswith(filename):
            for chunk in files_chunks[file]:
                to_delete = list()
                for line in chunk:
                    if match in line:
                        to_delete.append(line)
                for line in to_delete:
                    chunk.remove(line)
 def zap_line_in_file(filename, match):
    global header
    global files
@@ -195,7 +211,7 @@ def main():
        with open (filename, "r") as myfile:
            lines = myfile.readlines()
    else:
-        output = subprocess.check_output("git format-patch -1 --stdout", shell=True).decode("latin-1")
+        output = subprocess.check_output("git format-patch -1 --stdout", shell=True).decode("utf-8", errors="replace")
        lines = output.split("\n")
    parse_patch(lines)
@@ -209,6 +225,11 @@ def main():
    zap_entire_file_end("xz.sig")
    zap_entire_file_end("gz.sig")
    zap_entire_file_end("bz2.sig")
    zap_entire_file_end("xz.asc")
    zap_entire_file_end("gz.asc")
    zap_entire_file_end("bz2.asc")
    zap_entire_file_end(".pkey")
    zap_line_in_file("b/testresults", "Total : 0")
    zap_line_in_file("b/testresults", "Pass : 0")
@@ -223,15 +244,15 @@ def main():
    zap_line_in_file_start(".spec", "Release  :")
    zap_line_in_file_start(".spec", "Source99 :")
    zap_line_in_file_start(".spec", "Source0  :")
    zap_line_in_file_start(".spec", "export SOURCE_DATE_EPOCH")
    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
    zap_line_in_file_start(".spec", "export AR=gcc-ar")
    zap_line_in_file_start(".spec", "export RANLIB=gcc-ranlib")
    zap_line_in_file_start(".spec", "export NM=gcc-nm")
    zap_line_in_file_start(".spec", "export CFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export FCFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export FCFLAGS=\"$FFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export FFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export FFLAGS=\"$FFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export CXXFLAGS=\"$CXXFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "%setup -q -n ")
    zap_line_in_file_start(".spec", "URL      :")
@@ -265,12 +286,18 @@ def main():
    zap_line_in_file_start(".spec", "%description dev")
    zap_line_in_file_start(".spec", "%description license")
    zap_line_in_file_start(".spec", "%package doc")
    zap_line_in_file_start(".spec", "cp -a ")
    zap_line_in_file_start(".spec", "cd %{_builddir}/")
    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
    zap_line_in_file_start(".spec", "rm -rf %{buildroot}")
    zap_line_in_file_start("b/Makefile", "URL =")
    zap_line_in_file_start("b/Makefile", "PKG_NAME :=")
    zap_line_in_file_start("b/options.conf", "name = ")
    zap_line_in_file("b/options.conf", "archives = ")
    zap_line_in_file("b/options.conf", "giturl = ")
    zap_line_in_file_start("b/options.conf", "url = ")
    zap_line_in_file("b/options.conf", "[package]")
    zap_line_in_file("b/options.conf", "")
    zap_line_in_file("b/options.conf", "[autospec]")
@@ -287,11 +314,13 @@ def main():
    zap_line_in_file("b/options.conf", "# disable parallelization during build")
    zap_line_in_file("b/options.conf", "broken_parallel_build = false")
    zap_line_in_file("b/options.conf", "# this package is a library compatability package and only ships versioned library files")
    zap_line_in_file("b/options.conf", "# this package is a library compatibility package and only ships versioned library files")
    zap_line_in_file("b/options.conf", "compat = false")
    zap_line_in_file("b/options.conf", "# set conservative build flags")
    zap_line_in_file("b/options.conf", "conservative_flags = false")
    zap_line_in_file("b/options.conf", "# dev package requires the extras to be installed")
    zap_line_in_file("b/options.conf", "dev_requires_extras = false")
    zap_line_in_file("b/options.conf", "nodebug = false")
    zap_line_in_file("b/options.conf", "# pass -ffast-math to compiler")
    zap_line_in_file("b/options.conf", "fast-math = false")
    zap_line_in_file("b/options.conf", "# optimize build for speed over size")
@@ -321,9 +350,17 @@ def main():
    zap_line_in_file("b/options.conf", "# add clang flags")
    zap_line_in_file("b/options.conf", "use_clang = false")
    zap_line_in_file("b/options.conf", "# configure build for lto")
-    zap_line_in_file("b/options.conf", "use_lto = false")
+    zap_line_in_file("b/options.conf", "use_lto = true")
    zap_line_in_file("b/options.conf", "# require package verification for build")
    zap_line_in_file("b/options.conf", "verify_required = true")
    zap_line_in_file("b/options.conf", "# do not generate debuginfo for this package")
    zap_line_in_file("b/options.conf", "# configure build also for openmpi")
    zap_line_in_file("b/options.conf", "openmpi = false")
    zap_line_in_file("b/options.conf", "alias =")
    zap_line_in_file("b/options.conf", "alias = ")
    zap_line_in_file("b/options.conf", "nodebug = false")
    zap_line_in_file("b/options.conf", "domain =")
    zap_line_in_file("b/options.conf", "domain = ")
    zap_line_in_file("b/buildreq_add", "# This file contains additional build requirements that did not get")
    zap_line_in_file("b/buildreq_add", "# picked up automatically. One name per line, no whitespace.")
@@ -349,7 +386,11 @@ def main():
    zap_plus_line_in_file("symbols32")
    zap_entire_file("b/whatrequires")
    zap_entire_file("b/versions")
    zap_entire_file("b/.gitignore")
    zap_line_in_file_substring(".spec", "%{buildroot}/usr/share/package-licenses")   # version number change in license copy
    zap_empty_chunks()
    print_all()
@@ -102,7 +102,7 @@ fi
 required_progs() {
  local bindir="/usr/bin"
-  for f in git mock rpm rpmbuild ; do
+  for f in git mock rpm rpmbuild diffstat ; do
    [ ! -x "${bindir}/${f}" ] && missing+="${f} "
  done
  [ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
@@ -145,9 +145,9 @@ if [ "$USE_KOJI" ]; then
  cp "$CLIENTCA" ~/.koji/clientca.crt
  cp "$SERVERCA" ~/.koji/serverca.crt
-  if [ ! -f /etc/koji.conf ]; then
+  if [ ! -f ~/.koji/config ]; then
    echo "Setting up koji config . . ."
-    sudo cp projects/common/conf/koji.conf /etc
+    cp -f projects/common/conf/koji.conf ~/.koji/config
  fi
 fi