Update packages file for version 32550

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile

@@ -17,5 +17,9 @@ update:
 	curl -f -o packages ${BASE_URL}/source/package-sources
 	cut -f1 packages | LC_ALL=C sort > packages.new && mv packages.new packages
 
+.PHONY: spdx
 spdx:
-	curl -s spdx.org/licenses/ | sed '0,/<tbody>/d;/<\/tbody>/q;s/<tr>/\f/g;s/$$/,/g;s/<[^>]*>//g' | awk 'BEGIN{RS="\f";FS=","} {print $$4}' | sed '/^$$/d;s/^[ ]*//' > spdx
+	JSON=$$(mktemp); \
+	trap "rm $$JSON" EXIT; \
+	curl -f -S -s https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json > $$JSON || exit 1; \
+	jq -r '.licenses[] | .licenseId' < $$JSON | LC_COLLATE=C sort > licenses-spdx

Makefile.common

@@ -14,14 +14,14 @@ SPECFILE = $(addsuffix .spec, $(PKG_NAME))
 SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || rpmspec -D '_vendor clr' --srpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' $(SPECFILE))
 SRPMFILE = results/$(SRPMVERS).src.rpm
 
-LATEST_RPMS = $(shell [ ! -s results/build.log ] || for r in `egrep 'Wrote.*/RPMS/.*.rpm' results/build.log |  sed 's/.*\///'` ; do echo "results/$$r"; done)
+LATEST_RPMS = $(wildcard rpms/*.rpm)
+DEBUGINFO_RPM = $(wildcard rpms/*-debuginfo-*.rpm)
 RPMS ?= $(LATEST_RPMS)
 
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
 
 # MOCK_CONFIG_VAL is set in Makefile.shared
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL)
-
+MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_OPTS)
 
 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
@@ -29,38 +29,33 @@ __missingsources := $(filter-out $(wildcard ${__allsources}), ${__allsources})
 endif
 
 $(SPECFILE):
-	@echo Error: $(SPECFILE) is missing. Please run \`make autospec\` to try and auto
-	@echo Error: generate one for you.
-	@exit 1
+	@[[ -f "$@" ]] || (\
+	  echo Error: $@ is missing. Please run \`make autospec\` to try and auto;\
+	  echo Error: generate one for you.;\
+	  exit 1 )
 
 upstream:
-	@echo Error: The \"upstream\" file is missing. Try running \`make autospec\` to
-	@echo Error: auto generate one for you. If you are not using autospec to
-	@echo Error: generate a spec file, specify the proper "Source*" entries in the
-	@echo Error: spec header, and try running \`make generateupstream\`.
-	@exit 1
+	@[[ -f "$@" ]] || (\
+	  echo Error: The \"$@\" file is missing. Try running \`make autospec\` to;\
+	  echo Error: auto generate one for you. If you are not using autospec to;\
+	  echo Error: generate a spec file, specify the proper "Source*" entries in the;\
+	  echo Error: spec header, and try running \`make generateupstream\`.;\
+	  exit 1 )
 
 #help spdxcheck: Checks that the package license is a valid SPDX license,
 #help or that it is in the exceptions list.
 spdxcheck: $(SPECFILE)
 	@for LICENSE in `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{LICENSE}\n' $(SPECFILE)`; do \
-		if ! grep -q "^$$LICENSE$$" $(TOPLVL)/projects/common/licenses-spdx && ! grep -q "$$LICENSE$$" $(TOPLVL)/projects/common/licenses-exceptions; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
+		if ! grep -qx "$$LICENSE" $(TOPLVL)/projects/common/licenses-{spdx,exceptions,extra}; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
 	done; test -z "$$FAIL"
 
-#help cvecheck: Checks
-#help for common vulnerabilities and exposures in your package.
-cvecheck:
-	# https://github.com/ikeydoherty/cve-check-tool
-	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
-	cve-check-tool -M $(TOPLVL)/projects/common/mapping $(SPECFILE)
-
 #help clean: Cleans up the src.rpm file. but not the rpm files or log files.
 clean:
 	rm -f $(SRPMFILE)
 
 #help proper: Brings your copy of the package to a nearly clean git checkout state.
 proper: repodel
-	rm -rf results
+	rm -rf results rpms
 	rm -f build.log.round*
 	rm -f mock_build.log mock_srpm.log $(__allsources) report.html
 
@@ -68,10 +63,9 @@ proper: repodel
 mockclean:
 	$(MOCK) --clean --scrub=all
 
-#help mockproper: Clean mock chroot and cache directories for this package.
+#help mockproper: Clean mock chroot
 mockproper:
-	$(MOCK) --clean --scrub=all --uniqueext=$(PKG_NAME)
-	$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
+	$(MOCK) --clean --uniqueext=$(PKG_NAME)
 
 configemail:
 	@test -d .git || git init
@@ -89,8 +83,10 @@ prebuild-checks:
 #help files and rpms in the results directory of your package.  If there
 #help are mutlple tar.gz files for a package, make will do make sources,
 #help which creates a src.rpm file from the spec file.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options.
 build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
 	@$(MAKE) checkblacklist
@@ -98,6 +94,7 @@ build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
 #help build-nocheck: Same as 'make build', but do not run the package's test suite.
 build-nocheck: configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --nocheck --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
 	@$(MAKE) checkblacklist
@@ -115,9 +112,11 @@ shell:
 
 # Always rebuild the source rpm
 .PHONY: $(SRPMFILE)
-$(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice
+$(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice clean-old-content
 	@$(MAKE) spdxcheck
 	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	mv results/root.log results/srpm-root.log
+	mv results/build.log results/srpm-build.log
 
 # Do a git fetch and a git rebase to apply local commits on top of latest
 # commits from the remote. A git fetch/rebase is used rather than a git pull so
@@ -137,6 +136,10 @@ pullrebase:
 			echo "No commits exist in remote. Continuing."; \
 			exit 0; \
 		fi; \
+		if [ "$$(git rev-list FETCH_HEAD | head -n 1)" = "$$(git rev-list HEAD | head -n 1)" ]; then \
+			echo "Current branch up-to-date. Continuing."; \
+			exit 0; \
+		fi; \
 		echo "Rebasing to origin/master"; \
 		if git status --porcelain | grep -q '^.[^?]'; \
 		then \
@@ -148,37 +151,62 @@ pullrebase:
 		fi \
 	fi
 
+preautospec-checks:
+
 #help autospec: automatically generates a specfile. If there is
 #help already a specfile, it will be overwritten. Several files used by
-#help autospec will be created in the process. For more information, see
-#help the project at https://github.com/clearlinux/autospec
-autospec: pullrebase localreponotice
-	rm -f results/*src.rpm
+#help autospec will be created in the process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information, see the project at https://github.com/clearlinux/autospec
+autospec: preautospec-checks pullrebase localreponotice clean-old-content
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
-	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --integrity \
-		${NON_INTERACTIVE} --config "$(AUTOSPEC_CONF)" ${SKIP_GIT} \
-		$(firstword $(NEWURL) $(URL)) --name $(PKG_NAME) ${CLEANUP} \
-		$${SETVERSION:+ -v $${SETVERSION}} -a $(ARCHIVES) \
-		-m $(MOCK_CONFIG_VAL)
+	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
+		--target . \
+		--integrity \
+		--config "$(AUTOSPEC_CONF)" \
+		--name $(PKG_NAME) \
+		--archives $(ARCHIVES) \
+		--mock-config $(MOCK_CONFIG_VAL) \
+		--mock-opts="$(MOCK_OPTS)" \
+		$${SETVERSION:+ --version $${SETVERSION}} \
+		$${NON_INTERACTIVE:+ --non_interactive} \
+		$${SKIP_GIT:+ --skip-git} \
+		$${CLEANUP:+ -C} \
+		$(firstword $(NEWURL) $(URL));
+	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
 	@if [ -e update_changelog.sh ] && [ -z "$$SKIP_GIT" ] ; then \
 		bash update_changelog.sh ; \
 		git commit --amend --no-edit Change* ; \
 	fi
+	@$(MAKE) -s for-review.txt
+	diffstat for-review.txt
+	@printf "\n**\n"
+	@printf "** NOTICE: A patch with changes is available in the file for-review.txt\n"
+	@printf "** To recreate (e.g., after git commit --amend), run make for-review.txt\n"
+	@printf "** To submit for review: git send-email --to <recipient> for-review.txt\n"
+	@printf "**\n\n"
+
+#help for-review.txt: Creates the for-review.txt file, which is
+#help a minimized version of the Git commit, suitable for code review.
+for-review.txt:
+	git diff | grep -q index || python3 $(TOPLVL)/projects/common/patchfilter.py > for-review.txt
+.PHONY: for-review.txt
 
 #help autospecnogit: Runs autospec, but does not create a commit
 autospecnogit:
-	$(MAKE) SKIP_GIT=--skip-git autospec
+	$(MAKE) autospec SKIP_GIT=1
 
 #help autospecnostate: Runs autospec, but cleans up mock chroots
 #help and disables interactive mode.
 autospecnostate:
-	$(MAKE) CLEANUP=-C NON_INTERACTIVE=--non_interactive autospec
+	$(MAKE) autospec CLEANUP=1 NON_INTERACTIVE=1
 
 scanlicense:
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --config $(AUTOSPEC_CONF) --license-only $(firstword $(NEWURL) $(URL)) --name $(PKG_NAME)
@@ -187,7 +215,7 @@ scanlicense:
 bump:
 	git pull --rebase
 	$(MAKE) bumpnogit
-	git add *.spec release
+	git add $(SPECFILE) release
 	git commit -a -m "version bump from $(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)) to `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)`"
 
 #help bumpnogit: Increments the release file by one and does not commit changes.
@@ -200,30 +228,12 @@ bumpnogit:
 
 $(__missingsources): sources
 
-#help sources: If SOURCES_URL is defined, download required upstream source
-#help files from that location. Otherwise, try to extract source files from the
-#help SRPM in the latest Clear Linux release, as specified by the LATEST_SRPMS
-#help variable. This will run automatically, as a dependency. NOTE: A Koji
-#help server can make use of this "make sources" command, since it lives in a
-#help repo named "common". If you use this makefile in Koji, ensure
-#help "make sources" continues to work for both remote and local builds, since
-#help it is a prerequisite of several commands for building packages.
-sources: upstream
-ifneq ($(strip $(SOURCES_URL)),)
-	while read u; do \
-		case "$$u" in \
-		    "")    continue ;; \
-		    *://*) n="$$u" ;; \
-		    *)     n="$(SOURCES_URL)/$$u" ;; \
-		esac; \
-		curl --fail --noproxy "*" "$$n" -o `basename "$$n"`; \
-	done < upstream
-else
-	@tmp=$$(mktemp -d -p "."); \
+# First argument is a Clear Linux build number (e.g. 22000)
+define fetch-from-srpm
+	tmp=$$(mktemp -d -p "."); \
 	srpm=$$(rpmspec --srpm -q --qf "%{NVR}.src.rpm\n" $(SPECFILE)); \
-	if cd "$$tmp" && ! curl -f -L -O $(LATEST_SRPMS)/$$srpm; then \
-		echo "Failed to download $$srpm from latest Clear Linux release." >&2; \
-		echo "Run \"git pull\" and try again." >&2; \
+	if cd "$$tmp" && ! curl -s -S -f -L -O $(MIRROR_CURL_OPTS) $(DOWNLOAD_MIRROR)/releases/$(1)/clear/source/SRPMS/$$srpm; then \
+		echo "Failed to download $$srpm from Clear Linux release $(1)." >&2; \
 		cd .. && rm -rf "$$tmp"; \
 		exit 1; \
 	fi; \
@@ -231,14 +241,59 @@ else
 	cd ..; \
 	srclist=$$(rpmspec --srpm -q --qf "[%{SOURCE}\n]" $(SPECFILE)); \
 	for s in $$srclist; do \
-		if ! mv "$$tmp"/$$s .; then \
+		if ! mv --no-clobber "$$tmp"/$$s .; then \
 			echo "Missing source file \"$$s\" in $${srpm}." >&2; \
 			rm -rf "$$tmp"; \
 			exit 1; \
 		fi; \
 		echo "Retrieved source file $$s ..."; \
 	done; \
-	rm -rf "$$tmp";
+	rm -rf "$$tmp"
+endef
+
+#help sources: If SOURCES_URL is defined, download required upstream source files
+#help from that location. Otherwise, try to download source files according to the
+#help URLs listed in the spec file. If any of the source files fail to download,
+#help check for the relevant SRPM from published releases of Clear Linux OS, and
+#help extract the files if found. Note that SRPMs are taken from the DOWNLOAD_MIRROR
+#help location. This will run automatically, as a dependency. NOTE: A Koji server can
+#help make use of this "make sources" command, since it lives in a repo named
+#help "common". If you use this makefile in Koji, ensure "make sources" continues to
+#help work for both remote and local builds, since it is a prerequisite of several
+#help commands for building packages.
+ifneq ($(strip $(SOURCES_URL)),)
+sources: upstream
+	while read u; do \
+		case "$$u" in \
+			"")    continue ;; \
+			*://*) n="$$u" ;; \
+			*)     n="$(SOURCES_URL)/$$u" ;; \
+		esac; \
+		if ! curl --fail -L -o `basename "$$n"` $(SOURCES_CURL_OPTS) "$$n"; then \
+			echo "Failed to download $$n"; \
+			exit 1; \
+		fi; \
+	done < upstream
+else
+sources:
+	@$(MAKE) generateupstream; \
+	[ $$? -eq 0 ] && exit 0; \
+	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	git -C $(TOPLVL)/projects/common fetch --tags >/dev/null 2>&1; \
+	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
+	for b in $$latest_builds; do \
+		url="$(DOWNLOAD_MIRROR)/releases/$$b/clear/source/package-sources"; \
+		echo "Checking for source files in build $$b ..."; \
+		if grep -q "$$nvr" <(curl -s -f -L $(MIRROR_CURL_OPTS) $$url); then \
+			$(call fetch-from-srpm,$$b); \
+			if [ $$? -eq 0 ]; then \
+				echo "Source files retrieved from build $$b"; \
+				exit 0; \
+			fi; \
+		fi; \
+	done; \
+	echo "Source files not found for package"; \
+	exit 1
 endif
 
 prekoji-checks:
@@ -249,11 +304,22 @@ prekoji-checks:
 koji: prekoji-checks kojidef
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
-	@if ! git branch | grep -q -E '^\* master'; then \
+	@if [ "$$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then \
 		echo "Error: Must be on the master branch to submit to koji" >&2; \
 		exit 1; \
+	fi; \
+	if ! git diff --quiet HEAD -- ${SPECFILE}; then \
+		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
+		exit 1; \
+	fi; \
+	git fetch; \
+	if git rev-parse --verify --quiet origin/master >/dev/null; then \
+		git pull --rebase; \
+	fi; \
+	if git ls-remote --tags --exit-code origin $(SRPMVERS) >/dev/null; then \
+		echo "Error: remote tag $(SRPMVERS) already exists" >&2; \
+		exit 1; \
 	fi
-	git pull --rebase
 	git tag $(SRPMVERS)
 	git push origin master refs/tags/$(SRPMVERS)
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
@@ -270,7 +336,7 @@ scratch: $(SPECFILE) upstream $(SRPMFILE) kojidef
 
 #help scratch-wait: Performs scratch-build on package.
 scratch-wait: $(SPECFILE) upstream $(SRPMFILE) kojidef
-	$(KOJI_CMD) build --scratch $(KOJI_TAG) $(SRPMFILE)
+	$(KOJI_CMD) build --scratch --wait $(KOJI_TAG) $(SRPMFILE)
 
 #help koji-nowait: Same as 'make koji', but do not block
 koji-nowait:
@@ -291,22 +357,49 @@ $(TOPLVL)/repo:
 #help repoadd: Adds locally-built RPMs for this package to the local RPM
 #help repository. If this repository does not yet exist, it is created and
 #help enabled.
-repoadd: results/build.log $(TOPLVL)/repo
-	@if [ -z "`grep '^Wrote: .*$(shell basename $(SRPMFILE))$$' results/build.log`" ]; then \
-		echo "You have not yet succesfully built this release yet"; exit 1; \
-	fi
-	@if [ -f .repo-index ]; then \
+repoadd: $(TOPLVL)/repo
+	@if [ -z "${LATEST_RPMS}" ]; then \
+		echo "No rpms found in rpms/ directory."; \
+		exit 1; \
+	fi; \
+	if [ -f .repo-index ]; then \
 		echo "Cleaning old rpms:"; \
 		for r in `cat .repo-index`; do \
 			echo "-$$r"; \
 			rm -f $(TOPLVL)/repo/$$r; \
 		done; \
 		rm .repo-index; \
-	fi
-	for r in $(LATEST_RPMS); do echo "+$$r"; echo $$(basename $$r) >> .repo-index; cp $$r $(TOPLVL)/repo; done
+	fi; \
+	for r in ${LATEST_RPMS}; do \
+		echo "+$$r"; \
+		echo $$(basename $$r) >> .repo-index; \
+		ln -f $$r $(TOPLVL)/repo/; \
+	done
 	$(MAKE) localrepocreate
 	$(call localrepoenable,${PM_CONF},${MOCK_CONF})
 
+#help repostage: Stages locally-built RPMs for this package to the local RPM
+#help directory. If this directory does not yet exist, it is created and
+#help enabled.
+repostage: $(TOPLVL)/repo
+	@if [ -z "${LATEST_RPMS}" ]; then \
+		echo "No rpms found in rpms/ directory."; \
+		exit 1; \
+	fi; \
+	if [ -f .repo-index ]; then \
+		echo "Cleaning old rpms:"; \
+		for r in `cat .repo-index`; do \
+			echo "-$$r"; \
+			rm -f $(TOPLVL)/repo/$$r; \
+		done; \
+		rm .repo-index; \
+	fi; \
+	for r in ${LATEST_RPMS}; do \
+		echo "+$$r"; \
+		echo $$(basename $$r) >> .repo-index; \
+		ln -f $$r $(TOPLVL)/repo/; \
+	done
+
 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
 repodel:
@@ -355,25 +448,93 @@ install:
 	done
 	@$(MAKE) loop-down DEVICE=6
 
+#help install-debuginfo-local: Install locally built debuginfo RPM to
+#help the automatic debuginfo cache location (/var/cache/debuginfo)
+install-debuginfo-local:
+	@if [ -z "${DEBUGINFO_RPM}" ]; then \
+		echo "No debuginfo to install... skipping"; \
+		exit 0; \
+	fi; \
+	echo -n "Installing ${DEBUGINFO_RPM}... "; \
+	tmpdir=$$(mktemp -d); \
+	rpm2cpio ${DEBUGINFO_RPM} | ( cd $$tmpdir; cpio -i -d -u 2> /dev/null); \
+	dest=/var/cache/debuginfo/lib; \
+	find $$tmpdir/usr/lib/debug/ -mindepth 1 -maxdepth 1 2> /dev/null | while read -r d; do \
+		sudo chown -R dbginfo:dbginfo "$$d"; \
+		sudo cp -a "$$d" $$dest/; \
+	done; \
+	find $$tmpdir/usr/share/debug/ -mindepth 1 -maxdepth 1 -regextype awk -regex '.*/(.build-id|boot|lib|sbin|usr)$$' 2> /dev/null | while read -r d; do \
+		sudo chown -R dbginfo:dbginfo "$$d"; \
+		sudo cp -a "$$d" $$dest/; \
+	done; \
+	dest=/var/cache/debuginfo/src; \
+	find $$tmpdir/usr/src/debug/ -mindepth 1 -maxdepth 1 2> /dev/null | while read -r d; do \
+		sudo chown -R dbginfo:dbginfo "$$d"; \
+		sudo cp -a "$$d" $$dest/; \
+	done; \
+	find $$tmpdir/usr/share/debug/src/ -mindepth 1 -maxdepth 1 2> /dev/null | while read -r d; do \
+		sudo chown -R dbginfo:dbginfo "$$d"; \
+		sudo cp -a "$$d" $$dest/; \
+	done; \
+	sudo rm -rf $$tmpdir; \
+	echo "done"
+
+#help install-local: Install locally built RPMs to the root filesystem. Note that the
+#help debuginfo RPM installs to /var/cache/debuginfo
+install-local:
+	@for r in $(filter-out ${DEBUGINFO_RPM},${RPMS}); do \
+		echo -n "Installing $$r... "; \
+		rpm2cpio $$r | (cd /; sudo cpio -i -d -u 2> /dev/null); \
+		echo "done"; \
+	done
+	@$(MAKE) -s install-debuginfo-local
+
+#help install-mock: Install locally built RPMs to the mock rootcache
+#help filesystem. This command is usually used with "make shell".
+install-mock: repoadd
+	$(MOCK) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME) -i $(filter-out ${DEBUGINFO_RPM},${RPMS})
+
 #help generateupstream: Run this rule to create or update the 'upstream' file
 #help by downloading the upstream source tarballs listed in the spec file and
-#help calculating their hashes. Autospec performs this step automatically, so ignore
-#help it for packages managed with autospec.
+#help calculating their hashes. Autospec performs this step automatically, so
+#help ignore it for packages managed with autospec. Additional curl options
+#help may be used for downloading the upstream source tarballs by defining the
+#help CURL_OPTS variable for this rule.
 generateupstream:
-	@rm -f upstream
-	@urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | grep Source | cut -d: -f2- | grep '://'); \
+	@[ -e upstream ] && mv upstream upstream.bak; \
+	urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | sed -n "s|^Source[0-9]*[[:blank:]]*:[[:blank:]]*\(..*://..*\)$$|\1|p"); \
 	for url in $$urls; do \
-	    filename=$$(basename $$url); \
-	    if [ ! -e $$filename ]; then \
-		curl --fail -L -O $$url; \
-	    fi; \
-	    echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
+		filename=$$(basename $$url); \
+		if [ ! -e $$filename ]; then \
+			if ! curl --fail -L -O $(CURL_OPTS) $$url; then \
+				echo "Failed to download $$url"; \
+				[ -e upstream.bak ] && mv --no-clobber upstream.bak upstream; \
+				exit 1; \
+			fi; \
+		fi; \
+		echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
 	done
+	@rm -f upstream.bak
 	@cat upstream
 
 #help drop-abandoned: Remove all unused patches from the git tree
 drop-abandoned: $(SPECFILE)
 	python3 $(TOPLVL)/projects/common/drop-abandoned-patches.py $(SPECFILE)
 
+#help cloc: Count lines of code with the `cloc` tool on the full sources of package
+cloc: $(SRPMFILE)
+	@$(MOCK) -r $(TOPLVL)/repo/clear.cfg $(SRPMFILE) --result=results/ --no-cleanup-after
+	@$(MOCK) --chroot --copyin /usr/bin/cloc /usr/bin/cloc --result=results/ --no-cleanup-after
+	@$(MOCK) --result=results/ --no-cleanup-after --chroot '/usr/bin/cloc /builddir > /tmp/cloc.txt'
+	@$(MOCK) --copyout /tmp/cloc.txt results/ --result=results/
+	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
+	cat results/cloc.txt
+
+# Define LTS-specific targets in a separate makefile
+-include $(TOPLVL)/projects/common/Makefile.common.lts
+
 # Define site local common targets in a separate makefile
 -include $(TOPLVL)/projects/common/Makefile.common.site_local
+
+# Include optional extended makefiles from individual package repos
+-include Makefile.custom

Makefile.common.lts

@@ -0,0 +1,90 @@
+#-*-makefile-*-
+
+LTSUTILS = python $(TOPLVL)/projects/common/lts/main.py $(PKG_NAME)
+
+#help lts-show: Display a summary of active LTS branches.
+lts-show:
+	@$(LTSUTILS) sanity-check
+	@while read b; do \
+		if git show-ref $$b > /dev/null; then \
+			echo $$b $$(git log --oneline -1 $$b); \
+		else \
+			echo $$b Not found; \
+		fi; \
+	done < $(TOPLVL)/projects/common/lts/active-branches
+
+#help lts-backport: Fast-forward the previous active branch to the current
+#help branch.
+lts-backport:
+	@$(LTSUTILS) sanity-check
+	@newer=$$(git symbolic-ref HEAD); \
+	newer=$${newer#refs/heads/}; \
+	if ! $(LTSUTILS) prev-branch --checkout; then \
+		echo Could not check out previous active branch.; \
+		exit 0; \
+	fi; \
+	if $(LTSUTILS) is-same-version $$newer; then \
+		$(LTSUTILS) fast-forward $$newer; \
+	else \
+		echo Most likely a patch needs to be manually re-applied for this version.; \
+		echo Alternatively, use \"git merge --ff-only $$newer\" to upgrade the package version.; \
+	fi
+
+#help lts-build: Build RPM in Koji, or reuse existing build from older LTS
+#help branch.
+lts-build:
+	@$(LTSUTILS) sanity-check
+	@tag=$$(git tag --points-at); \
+	if [[ -z $$tag ]]; then \
+		echo This commit does not have a tag.; \
+		echo Proceeding to run \"make lts-koji\".; \
+		$(LTSUTILS) prompt || exit; \
+		$(MAKE) lts-koji; \
+		exit; \
+	fi; \
+	older=$$($(LTSUTILS) prev-branch); \
+	if [[ -z $$older ]]; then \
+		echo Could not determine previous active branch.; \
+		exit 1; \
+	fi; \
+	current=$$($(LTSUTILS) current-branch); \
+	if [[ $$(git rev-parse $$older) != $$(git rev-parse $$current) ]]; then \
+		echo Previous branch and current branch have different commits.; \
+		for b in $$older $$current; do echo $$b $$(git log --oneline -1 $$b); done; \
+		echo Proceeding to run \"make lts-koji\".; \
+		$(LTSUTILS) prompt || exit; \
+		$(MAKE) lts-koji; \
+		exit; \
+	fi; \
+	if $(LTSUTILS) can-reuse-binary $$older; then \
+		echo Reusing binary from $$older.; \
+		$(KOJI_CMD) tag-build $$current $(SRPMVERS); \
+	else \
+		echo Could not reuse binary from $$older.; \
+		echo Proceeding to run \"make bump lts-koji\".; \
+		$(LTSUTILS) prompt || exit; \
+		$(MAKE) bump lts-koji; \
+	fi
+	@$(LTSUTILS) next-branch --checkout
+
+#help lts-koji: Same as "koji" but for working on LTS branches. Should not
+#help be called directly, use lts-build instead.
+lts-koji: prekoji-checks kojidef
+	@$(LTSUTILS) sanity-check
+	@$(MAKE) spdxcheck
+	@$(MAKE) checkblacklist
+	@current=$$($(LTSUTILS) current-branch); \
+	if ! grep $$current $(TOPLVL)/projects/common/lts/active-branches; then \
+		echo "Error: Must be on an active branch to submit to koji" >&2; \
+		exit 1; \
+	fi; \
+	if ! git diff --quiet HEAD ${SPECFILE}; then \
+		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
+		exit 1; \
+	fi; \
+	if git rev-parse --verify --quiet origin/$$current > /dev/null; then \
+		git pull --rebase; \
+	fi
+	git tag $(SRPMVERS)-lts
+	git push origin $$($(LTSUTILS) current-branch) refs/tags/$(SRPMVERS)-lts
+	$(KOJI_CMD) build $$KOJI_NOWAIT $$($(LTSUTILS) current-branch) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)-lts

Makefile.config

@@ -26,6 +26,9 @@ CGIT_BASE_URL =
 # Location of upstream tarball cache used by 'make build', 'make sources', etc.
 SOURCES_URL =
 
+# Additional curl options to use when downloading files from SOURCES_URL
+SOURCES_CURL_OPTS =
+
 # Location where package repos are hosted.
 PKG_BASE_URL = https://github.com/clearlinux-pkgs
 
@@ -38,6 +41,9 @@ PROJ_PKGS = autospec clr-bundles common
 # Clear Linux download mirror to use
 DOWNLOAD_MIRROR = https://cdn.download.clearlinux.org
 
+# Additional curl options to use when downloading files from DOWNLOAD_MIRROR
+MIRROR_CURL_OPTS =
+
 # Location of release content from latest Clear Linux release
 LATEST_RELEASE = $(DOWNLOAD_MIRROR)/current
 
@@ -48,17 +54,17 @@ LATEST_SRPMS = $(LATEST_RELEASE)/source/SRPMS
 PACKAGES_FILE = $(TOPLVL)/projects/common/packages
 
 # Configuration file for autospec.
-AUTOSPEC_CONF = $(TOPLVL)/projects/common/autospec.conf
+AUTOSPEC_CONF = $(TOPLVL)/projects/common/conf/autospec.conf
 
 # Set this variable to any non-empty value to use the package mappings stored
 # in "pkg-mapping" for cloning/pulling package repos.
 USE_PACKAGE_MAPPING = yes
 
-# Package manager config file. Default to the in-tree copy.
-PM_CONF = $(TOPLVL)/projects/common/image-creator/yum.conf
+# Package manager config file. Default to the in-tree copy of dnf.conf.
+PM_CONF = $(TOPLVL)/projects/common/conf/dnf.conf
 
 # Mock configuration file. Default to the in-tree copy.
-MOCK_CONF = $(TOPLVL)/projects/common/koji-client-files/clear.cfg
+MOCK_CONF = $(TOPLVL)/projects/common/conf/clear.cfg
 
 
 # Include any site local configuration variables. These variables may override

Makefile.shared

@@ -14,9 +14,9 @@ define loopup
 	@sudo partprobe /dev/loop$(DEVICE)
 	@sleep 1
 	@if [ -e /dev/loop$(DEVICE)p3 ]; then \
-	   sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
 	else \
-	   sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
 	fi
 	@sudo mount /dev/loop$(DEVICE)p1 $(TOPLVL)/image/boot
 endef
@@ -68,10 +68,10 @@ endef
 # augmented for local repo support.
 define localrepoenable
 	@cp $(1) $(TOPLVL)/repo/yum.conf
-	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/yum.conf
+	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $(TOPLVL)/repo/yum.conf
 	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/yum.conf
 	@cp $(2) $(TOPLVL)/repo/clear.cfg
-	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/clear.cfg
+	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $(TOPLVL)/repo/clear.cfg
 	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/clear.cfg
 	@sed -i '/^"""/d; $$a"""' $(TOPLVL)/repo/clear.cfg
 endef
@@ -138,6 +138,30 @@ localreponotice:
 		[ -f $(TOPLVL)/repo/opt-in-build ] || sleep 3; \
 	fi >&2
 
+.PHONY: clean-old-rpms clean-old-logs clean-old-content
+clean-old-rpms:
+	rm -rf ./rpms
+	rm -f ./results/*.rpm
+
+clean-old-logs:
+	rm -f ./results/*.log
+
+clean-old-content: clean-old-rpms clean-old-logs
+
+.PHONY: require-pkg-repo-dir link-new-rpms
+require-pkg-repo-dir:
+	@if [ -z ${PKG_REPO_DIR} ]; then \
+		echo "Please specify PKG_REPO_DIR to indicate package repo location."; \
+		exit 1; \
+	fi
+
+link-new-rpms: require-pkg-repo-dir
+	mkdir -p ${PKG_REPO_DIR}/rpms
+	rm -f ${PKG_REPO_DIR}/rpms/*.rpm
+	find ${PKG_REPO_DIR}/results -maxdepth 1 -name '*.rpm' -exec ln {} ${PKG_REPO_DIR}/rpms/ \;
+	rm -f ${PKG_REPO_DIR}/rpms/*.src.rpm
+
+
 .PHONY: help
 ##### Code
 # Make sure that HELPSPACE has exactly HELPLEN spaces in it

Makefile.toplevel

@@ -10,6 +10,8 @@ include $(TOPLVL)/projects/common/Makefile.shared
 SHELL   = /bin/bash
 
 PKGS    = $(shell cat $(PACKAGES_FILE))
+PKGS_LOCAL = $(sort $(filter ${PKGS},$(subst packages/,,$(wildcard packages/*))))
+PKGS_ORPHANED = $(sort $(filter-out common ${PKGS},$(subst packages/,,$(wildcard packages/*))))
 
 proj_PKGS = $(addprefix proj_,$(PROJ_PKGS))
 clone_PKGS = $(addprefix clone_,$(PKGS))
@@ -32,7 +34,7 @@ $(proj_PKGS):
 	git clone $(PRJ_BASE_URL)/$$proj projects/$$proj; \
 	cd projects/$$proj; \
 	if [ "$$proj" = "autospec" ] ; then \
-		git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+		git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 	else \
 		$(call gitoliteurl,projects/$$proj); \
 	fi; \
@@ -54,17 +56,11 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	$(call subjectprefix,$$pkg); \
 	)
 
-#help cvecheck: Checks for common vulnerabilities and exposures in your code.
-cvecheck:
-	# https://github.com/ikeydoherty/cve-check-tool
-	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
-	cve-check-tool -n -M $(TOPLVL)/projects/common/mapping $(PACKAGES_FILE)
-
 #help pull: Performs a git pull --rebase for each package repo, avoiding the creation
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
 .PHONY: pull ${PULL_PKGS} 
-PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,$(PKGS))
+PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,${PKGS_LOCAL})
 ${PULL_PKGS}:
 	@p=$(patsubst PULL_%,%,$@) ; \
 	if [ ! -d "$$p/.git" ]; then echo "Nothing to pull for $$p - $$p/.git missing"; exit 0; fi; \
@@ -81,7 +77,7 @@ ${PULL_PKGS}:
 			git --no-pager log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit $$O..$$N; echo; \
 		fi ; \
 		if [ "$$p" = "projects/autospec" ] ; then \
-			git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+			git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 		else \
 			$(call gitoliteurl,$$p); \
 		fi ; \
@@ -127,19 +123,17 @@ pull: ${PULL_PKGS}
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
 #help removes the associated package repos from the ./packages tree
 clean-pkgs-dir: $(PACKAGES_FILE)
-	@for p in `ls packages`; do \
-		if ! grep -wq "^$$p$$" "$<" && [ "$$p" != common ]; then \
-			echo "Removing $$p from packages, it is no longer in common/packages."; \
-			rm -rf packages/$$p; \
-		fi \
+	@for p in ${PKGS_ORPHANED}; do \
+		echo "Removing $$p from packages, it is no longer in common/packages."; \
+		rm -rf packages/$$p; \
 	done
 	@echo "packages directory cleaned";
 
-clean_PKGS = $(addprefix clean_,$(PKGS))
+clean_PKGS = $(addprefix clean_,$(PKGS_LOCAL))
 #help clean: Run 'make clean' for every package.
 clean: $(clean_PKGS)
 
-proper_PKGS = $(addprefix proper_,$(PKGS))
+proper_PKGS = $(addprefix proper_,$(PKGS_LOCAL))
 #help proper: Run 'make proper' for every package, and purge the local
 #help repo and image.
 proper: $(proper_PKGS)
@@ -153,34 +147,34 @@ proper: $(proper_PKGS)
 
 $(clean_PKGS):
 	@echo "cleaning $(patsubst clean_%,%,$@)"
-	-@$(MAKE) -s -j -C $(addprefix packages/,$(patsubst clean_%,%,$@)) clean
+	-@$(MAKE) -s -C $(addprefix packages/,$(patsubst clean_%,%,$@)) clean
 
 $(proper_PKGS):
-	-@$(MAKE) -s -j -C $(addprefix packages/,$(patsubst proper_%,%,$@)) proper
+	-@$(MAKE) -s -C $(addprefix packages/,$(patsubst proper_%,%,$@)) proper
 
 #help status: Runs git status for all package repos, thus displaying untracked
 #help and unstaged files in addition to staged files.
-status: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS))
-	@for p in projects/common $(addprefix packages/,$(PKGS)); do \
+status: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
+	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
 	if [ -d "$$p/.git" ] &&  [ -n "$$(git -C $$p status -uno --porcelain)" ]; then echo "Uncommitted changes in $$p:"; git -C "$$p" status --short; fi ;\
 	done
 
 #help diff: Runs git diff for all package repos and displays the output using
 #help diffstat. Requires that diffstat is installed.
-diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS))
-	@for p in projects/common $(addprefix packages/,$(PKGS)); do \
+diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
+	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
 		(cd $$p ; git status | grep -q 'nothing to commit, working [^ ]* clean' || (echo "Uncommitted changes in: $$p"; git diff |diffstat -p1)) ;\
 	done
 
 #help versions: Displays the version of each package in the ./packages tree.
-versions: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS))
-	@for p in $(addprefix packages/,$(PKGS)); do \
+versions: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
+	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
 		(cd $$p; if ls *.spec &> /dev/null; then rpmspec -q --queryformat '%{NAME}-%{VERSION}\n' *.spec | head -n1; fi) ;\
 	done
 
 #help releases: Like 'make versions', but also displays the release number.
-releases: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS))
-	@for p in $(addprefix packages/,$(PKGS)); do \
+releases: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
+	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
 		(cd $$p; if ls *.spec &> /dev/null; then rpmspec -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' *.spec | head -n1; fi) ;\
 	done
 
@@ -189,14 +183,20 @@ releases: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS))
 provides:
 	@$(TOPLVL)/projects/common/provides.sh -f $(FP) -r $(RN)
 
+preautospecnew-checks:
+
 #help autospecnew: Creates a new autospec package with for a given URL=$(URL)
 #help with NAME=$(NAME). Several files used by autospec will be created in the
-#help process. For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
-autospecnew: localreponotice
+#help process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
+autospecnew: preautospecnew-checks localreponotice
 	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
+	-$(MAKE) clone_$(NAME)
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
@@ -208,8 +208,74 @@ autospecnew: localreponotice
 			$(call subjectprefix,$(NAME)); \
 		); \
 		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
-		python3 $(TOPLVL)/projects/autospec/autospec/autospec.py --integrity -t packages/$(NAME) --config $(AUTOSPEC_CONF) $(URL) --name $(NAME) -m $(MOCK_CONFIG_VAL); \
-		$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
+		python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
+			--target packages/$(NAME) \
+			--integrity \
+			--config "$(AUTOSPEC_CONF)" \
+			--name $(NAME) \
+			--archives $(ARCHIVES) \
+			--mock-config $(MOCK_CONFIG_VAL) \
+			--mock-opts="$(MOCK_OPTS)" \
+			$${SETVERSION:+ --version $${SETVERSION}} \
+			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+			$(URL); \
+		if [ $$? -eq 0 ]; then \
+			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
+			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
+			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) format-patch -1 --stdout) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
+			printf "\n**\n"; \
+			printf "** NOTICE: A patch with changes is available in the file $(TOPLVL)/packages/$(NAME)/for-review.txt\n"; \
+			printf "** Please look through this file and if you are unsure, please submit for code review with git send-email\n"; \
+			printf "**\n\n"; \
+		else \
+			echo "Autospec of $(NAME) failed."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
+		exit 1; \
+	fi
+
+#help autospecnewgo: Creates a new autospec package with (dependencies)
+#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
+#help autospec will be created in the help process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
+autospecnewgo: preautospecnew-checks localreponotice
+	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
+		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
+		exit 1; \
+	fi
+	-$(MAKE) clone_$(NAME)
+	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
+		echo "no remote repository found, creating new package repository and running autospec"; \
+		mkdir -p $(TOPLVL)/packages/$(NAME); \
+		( \
+			cd $(TOPLVL)/packages/$(NAME); \
+			git init; \
+			git remote add origin $(PKG_BASE_URL)/$(NAME); \
+			$(call gitoliteurl,packages/$(NAME)); \
+			$(call subjectprefix,$(NAME)); \
+		); \
+		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
+		python3 $(TOPLVL)/projects/common/go-builder.py \
+			--target packages/$(NAME) \
+			--integrity \
+			--config "$(AUTOSPEC_CONF)" \
+			--name $(NAME) \
+			--archives $(ARCHIVES) \
+			--mock-config $(MOCK_CONFIG_VAL) \
+			--mock-opts "$(MOCK_OPTS)" \
+			$(URL); \
+		if [ $$? -eq 0 ]; then \
+			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
+			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
+			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
+			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
+		else \
+			exit 1; \
+		fi; \
 	else \
 		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
 		exit 1; \

README-advanced.md

@@ -0,0 +1,40 @@
+## Advanced topics
+
+### Manual setup
+
+If you did not run the user-setup script (see "Automated setup" section in the
+[main README](README.md), you will want to set up the developer tooling
+workspace manually. This section provides general documentation for the manual
+setup process, and it is not meant to be exhaustive.
+
+On your Clear Linux system, create a workspace for Clear Linux development
+work:
+
+```
+$ mkdir clearlinux
+```
+
+Clone this repo into a `projects` directory within the workspace:
+
+```
+$ cd clearlinux
+$ mkdir projects
+$ git clone https://github.com/clearlinux/common projects/common
+```
+
+Create the toplevel tooling Makefile:
+
+```
+$ ln -s projects/common/Makefile.toplevel Makefile
+```
+
+Clone all Clear Linux package and project repositories:
+
+```
+$ make clone
+```
+
+Note: You can clone the repos in parallel by using make's `-j` option.
+
+At this point, the `packages` directory will contain all Clear Linux package
+repos, and `projects` will contain common, clr-bundles, and autospec repos.

README.md

@@ -31,43 +31,13 @@ $ ./user-setup.sh
 After the script completes, make sure to logout and login again to complete the
 setup process.
 
-The script either accepts no options, or all (3) options in case you are
-configuring the Koji CLI for remote building on a Koji server. The options are
-documented in the script's `--help` output.
+The script accepts several options, or no options at all. The options are
+documented in the script's `--help` output. Note that if you are supplying any
+of the three Koji cert options (`-k`, `-s`, or `-c`), the other two options
+must be supplied as well.
 
-### Manual setup
-
-On your Clear Linux system, create a workspace for Clear Linux development
-work:
-
-```
-$ mkdir clearlinux
-```
-
-Clone this repo into a `projects` directory within the workspace:
-
-```
-$ cd clearlinux
-$ mkdir projects
-$ git clone https://github.com/clearlinux/common projects/common
-```
-
-Create the toplevel tooling Makefile:
-
-```
-$ ln -s projects/common/Makefile.toplevel Makefile
-```
-
-Clone all Clear Linux package and project repositories:
-
-```
-$ make clone
-```
-
-Note: You can clone the repos in parallel by using make's `-j` option.
-
-At this point, the `packages` directory will contain all Clear Linux package
-repos, and `projects` will contain common, clr-bundles, and autospec repos.
+If you do not wish to run the user-setup script, see the "Manual setup" section
+below for hints about how to initialize the tooling workspace.
 
 ## Example usage
 
@@ -132,6 +102,10 @@ $ echo missing-build-req >> buildreq_add
 $ make autospec
 ```
 
+Please see https://github.com/clearlinux/autospec#common-files for 
+documention on buildreq_add and the other files autospec uses during the
+build process.
+
 ### Bump the release number for a package
 
 If you simply need to increment a package's release number and rebuild the
@@ -141,3 +115,41 @@ package, a `make bump` command is available for this purpose.
 $ make bump
 $ make build
 ```
+
+### Update the release version for a package
+
+If you have an update release version for a package, you can change the url
+for the new release in the package/example-pkg/Makefile.  After modifying the
+new url, run `make autospec` again to fetch the new package and rebuild.
+
+```
+$ make autospec
+```
+
+## Other topics
+
+### Customizing the mock config
+
+In the past, the various make commands that call `mock` required a mock config
+installed at `/etc/mock/clear.cfg`. However, at present, the commands will
+instead use the mock config within this repo (`conf/clear.cfg`).
+
+If you wish to use a custom mock config, you must override the `MOCK_CONF`
+variable to specify a different value to pass to mock's `-r` option. The value
+is either a full path that ends with `.cfg`, or a config NAME installed at
+`/etc/mock/<NAME>.cfg`. You can override the `MOCK_CONF` config variable by
+redefining it in `Makefile.config.site_local`, which must reside at the
+toplevel directory in this repo.
+
+For example, to retain the old behavior of mock using `/etc/mock/clear.cfg`,
+add this line to `Makefile.config.site_local`:
+
+```
+MOCK_CONF = /etc/mock/clear.cfg
+```
+
+If `Makefile.config.site_local` doesn't exist already, create it.
+
+### Manual setup
+
+See the [Manual setup](README-advanced.md#manual-setup) documentation.

conf/README.md

@@ -0,0 +1,31 @@
+# Configuration files
+
+This directory contains various configuration files used by the developer
+tooling framework.
+
+Unless otherwise noted in the config file documentation below, the config files
+can be modified with custom, site-local changes by modifying a variable in
+`Makefile.config.site_local` to point to a location of your choice (e.g.
+somewhere in `/etc`). The framework will then read the config file from the
+location you have specified instead, so copy the in-tree copy to that location
+as a starting point if you need to.
+
+* `autospec.conf`: default Autospec configuration file used by `make autospec`
+  and `make autospecnew`. All configuration is commented out by default, so
+  default Autospec settings are used. Config variable: `AUTOSPEC_CONF`
+
+* `clear.cfg`: default Mock configuration file used by various targets that
+  call out to Mock. Config variable: `MOCK_CONF`
+
+* `dnf.conf`: DNF conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `yum.conf`: YUM conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `koji.conf`: template Koji configuration file. It is installed by
+  `user-setup.sh` to `/etc/koji.conf`, and the in-tree copy is not used. If you
+  are using Koji with the framework, you will need to modify `/etc/koji.conf`
+  according to how the Koji instance is set up for your environment. Configuring
+  the installation location via `Makefile.config.site_local` is not possible at
+  the moment but is a planned feature.

conf/autospec.conf

@@ -18,6 +18,10 @@
 # an absolute path or a path relative to this autospec.conf.
 #packages_file =
 
-# Path to yum config, used for generating whatrequires. It can be an absolute
+# Path to dnf/yum config, used for generating whatrequires. It can be an absolute
 # path or a path relative to this autospec.conf.
 #yum_conf =
+
+# Path to autospec failed patterns directory, used to match build failure lines to
+# package dependencies.
+#failed_pattern_dir =

conf/clear.cfg

@@ -6,17 +6,22 @@ config_opts['dist'] = 'clear'  # only useful for --resultdir variable subst
 config_opts['extra_chroot_dirs'] = [ '/run/lock', ]
 config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s  %(user)s'
 config_opts['plugin_conf']['ccache_enable'] = False
+config_opts['plugin_conf']['root_cache_opts']['compress_program'] = 'zstd -5'
+config_opts['plugin_conf']['root_cache_opts']['extension'] = '.zst'
 config_opts['releasever'] = 'clear'
 config_opts['package_manager'] = 'dnf'
+config_opts['chrootgroup'] = 'mockbuild'
+config_opts['use_bootstrap_container'] = False
 
 
-config_opts['yum.conf'] = """
+config_opts['dnf.conf'] = """
 
 [main]
-cachedir=/var/cache/yum
+cachedir=/var/cache/dnf
+keepcache=1
 debuglevel=1
 reposdir=/dev/null
-logfile=/var/log/yum.log
+logfile=/var/log/dnf.log
 retries=20
 obsoletes=1
 gpgcheck=0

conf/dnf.conf

@@ -0,0 +1,24 @@
+[main]
+keepcache=1
+debuglevel=1
+logfile=/var/log/dnf.log
+exactarch=1
+obsoletes=1
+gpgcheck=0
+plugins=0
+retries=10
+installonly_limit=3
+reposdir=/root/mash
+
+[clear]
+name=Clear
+baseurl=https://cdn.download.clearlinux.org/current/x86_64/os/
+enabled=1
+gpgcheck=0
+
+[clear-source]
+name=Clear sources
+failovermethod=priority
+baseurl=https://cdn.download.clearlinux.org/current/source/SRPMS/
+enabled=1
+gpgcheck=0

fmvpatches.pl

@@ -82,7 +82,7 @@ sub find_file {
     return $file;
 }
 
-open(BUILD_LOG, '<', "results/build.log") or die $!;
+open(BUILD_LOG, '<', "results/logs/build.log") or die $!;
 while (<BUILD_LOG>) {
     if($_ =~ /(\S+):([0-9]*):([0-9]*): note: (basic block|loop) (vectorized)/) {
 	$fmv->{s_name} = (split('/',$1))[-1];

go-builder.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+#
+# gowrap.py - part of autospec
+# Copyright (C) 2019 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def parse_args():
+    """Parse commandline arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url", help="Location of package archive")
+    parser.add_argument("-t", "--target", dest="target", action="store",
+                        default=None,
+                        help="Target location to create or reuse")
+    parser.add_argument("-c", "--config", dest="config", action="store",
+                        default="/usr/share/defaults/autospec/autospec.conf",
+                        help="Set configuration file to use")
+    parser.add_argument("-n", "--name", action="store", dest="name", default="",
+                        help="Override the package name")
+    parser.add_argument("-i", "--integrity", action="store_true",
+                        default=False,
+                        help="Search for package signature from source URL and "
+                             "attempt to verify package")
+    parser.add_argument("-m", "--mock-config", action="store", default="clear",
+                        help="Value to pass with Mock's -r option. Defaults to "
+                             "\"clear\", meaning that Mock will use "
+                             "/etc/mock/clear.cfg.")
+    parser.add_argument("-o", "--mock-opts", action="store", default="",
+                        help="Arbitrary options to pass down to mock when "
+                        "building a package.")
+    parser.add_argument('-a', "--archives", action="store",
+                        dest="archives", default=[], nargs='*',
+                        help="tarball URLs for additional source archives and"
+                        " a location for the sources to be extacted to (e.g."
+                        " http://example.com/downloads/dependency.tar.gz"
+                        " /directory/relative/to/extract/root )")
+
+    return parser.parse_args()
+
+
+def missing_dependencies(name):
+    """Check if the build failed due to missing dependencies."""
+    root_log = os.path.join("packages", name, "results", "root.log")
+    if not os.path.exists(root_log):
+        return False
+    with open(root_log, "r") as lfile:
+        for line in lfile.readlines():
+            if "No matching package to install" in line:
+                return True
+    return False
+
+
+def already_built(name, version):
+    """Check if the package and version are already built."""
+    if not os.path.exists(os.path.join("packages", name, "rpms")):
+        return False
+    vpath = os.path.join("packages", name, "versions")
+    if not os.path.exists(vpath):
+        return False
+    with open(vpath, "r") as vfile:
+            if version not in [x.strip() for x in vfile.readlines()]:
+                return False
+    return True
+
+
+def build(path, args, name, url, version):
+    """Try and build a package."""
+    if version:
+        ver_arg = ["-v", version]
+    else:
+        ver_arg = []
+    if already_built(name, version):
+        return True
+    print(f"Trying to autospec {name} - {version}")
+
+    proc = subprocess.run(["python3",
+                           f"{path}/../autospec/autospec/autospec.py", url,
+                           "-c", args.config, "-t", f"packages/{name}",
+                           "-n", name, "-m", args.mock_config,
+                           "-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
+    if proc.returncode == 0:
+        subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
+                       capture_output=True)
+    return proc.returncode == 0
+
+
+def parse_go_mod(path):
+    """Parse go.mod file for build requirements.
+
+    File content looks as follows:
+
+    module example.com/foo/bar
+
+    require (
+        github.com/BurntSushi/toml v0.3.1
+        git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
+        github.com/inconshreveable/mousetrap v1.0.0 // indirect
+        "github.com/spf13/cobra" v0.0.3
+        github.com/spf13/pflag v1.0.3 // indirect
+    )
+
+    Need to handle all require lines including //indirect.
+    Skip requires that use .git for now. May need to be handled
+    differently.
+    """
+    reqs = []
+    with open(path, "r") as gfile:
+        dep_start = False
+        for line in gfile.readlines():
+            # Ideally the mod file is generated and the format is
+            # always correct but add a few defenses just in case
+            line = line.strip()
+            if line.startswith("//"):
+                # Skip comments
+                continue
+            if dep_start:
+                # End of the require section
+                if line.startswith(")"):
+                    break
+                req = line.split()[:2]
+                req[0] = req[0].replace('"', '')
+                if req[0].endswith(".git"):
+                    continue
+                reqs.append(req)
+                continue
+            if line.startswith("require ("):
+                dep_start = True
+    return reqs
+
+
+def get_dependencies(name):
+    """Return path to the go.mod file if it exists."""
+    command = None
+    path = None
+    pdir = os.path.join("packages", name)
+    reqs = []
+    for fname in os.listdir(pdir):
+        # Find the archive
+        if fname.endswith("zip"):
+            command = ["unzip", fname]
+        elif ".tar." in fname:
+            command = ["tar", "xf", fname]
+        else:
+            command = None
+
+        if command:
+            # Decompress archive
+            tdir = tempfile.mkdtemp()
+            tfile = os.path.join(tdir, fname)
+            shutil.copyfile(os.path.join(pdir, fname), tfile)
+            proc = subprocess.run(command, cwd=tdir, capture_output=True)
+            if not proc.returncode:
+                # inspect contents for a "go.sum" file
+                for root, _, files in os.walk(tdir):
+                    if "go.mod" in files:
+                        path = f"{name}.gomod"
+                        shutil.copyfile(os.path.join(root, "go.mod"), path)
+                        reqs += parse_go_mod(path)
+                        os.unlink(path)
+            shutil.rmtree(tdir)
+
+    return reqs
+
+
+def encode_requirements(requirement):
+    """Encode the project's module and version."""
+    # Default dependency base proxy url
+    base_url = "https://proxy.golang.org/"
+    # Encoding based on
+    # https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
+    encoded_module = ""
+    for char in requirement[0]:
+        if char.isupper():
+            encoded_module += "!" + char.lower()
+        else:
+            encoded_module += char
+    encoded_version = ""
+    for char in requirement[1]:
+        if char.isupper():
+            encoded_version += "!" + char.lower()
+        else:
+            encoded_version += char
+    return (encoded_module, encoded_version)
+
+
+def initialize_package(name, url, version, path, args):
+    """Setup package and try to build it."""
+    if already_built(name, version):
+        return True
+    print(f"Trying to autospecnew {name} - {version}")
+
+    proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
+    if f"{name} already exists at" in proc.stdout.decode("utf-8"):
+        return build(path, args, name, url, version)
+    return proc.returncode == 0
+
+
+def build_recursive(path, args, name, url, version=None, success=None):
+    """Try and recursively build packages."""
+    if not success:
+        # First package being built, common utilities already run
+        # so just use autospec directly
+        success = {}
+        ret = build(path, args, name, url, version)
+    else:
+        ret = initialize_package(name, url, version, path, args)
+
+    if ret:
+        # Completed so we are done!
+        return True
+
+    if not missing_dependencies(name):
+        # Failed but not due to missing dependencies, bail
+        print(f"Unknown error building {name} - {version}")
+        return False
+
+    reqs = get_dependencies(name)
+    if not reqs:
+        # Didn't try and add anything so don't need to build again
+        print(f"Build failed with no missing requirements: {name} - {version}")
+        return False
+
+    print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
+    for req in reqs:
+        ereq = encode_requirements(req)
+        rname = "go-" + req[0].replace("/", "-")
+        rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
+        if (rname, req[1]) in success:
+            # Rebuild detected, figure out if it is okay or not
+            if success[(rname, req[1])]:
+                # Already succeeded building req short circuit success
+                continue
+            else:
+                # Encountered a build loop, bail out as this likely
+                # requires manual fixing
+                print(f"Detected build loop when building {rname} - {req[1]}")
+                return True
+        success[(rname, req[1])] = False
+        if not build_recursive(path, args, rname, rurl, req[1], success):
+            print(f"Failed to build dependency {rname} - {req[1]}")
+            return False
+
+        success[(rname, req[1])] = True
+        subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
+        subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
+
+    # Create the repo with the new packages added
+    subprocess.run(["make", "localrepocreate"], capture_output=True)
+    # Retry previously failed build assuming dependencies got added
+    print(f"Rebuilding after resolving dependencies {name} - {version}")
+    ret = build(path, args, name, url, version)
+    print("")
+
+    return ret
+
+
+def main():
+    """Start program execution."""
+    args = parse_args()
+    path = os.path.dirname(os.path.realpath(__file__))
+    return build_recursive(path, args, args.name, args.url)
+
+if __name__ == '__main__':
+    if not main():
+        sys.exit(-1)
+    print("\n\nBuild completed\n\n")

licenses-exceptions

@@ -1,4 +1,5 @@
 Copyright
 Distributable
+ISSL
 Public-Domain
 sendmail

licenses-extra

@@ -0,0 +1,7 @@
+# This file contains SPDX license identifiers that are not defined in the
+# latest version of SPDX but still needed (or possibly needed) for Clear Linux
+# packages.
+GFDL-1.3+
+MIT-Opengroup
+WXwindows
+w3c

licenses-spdx

@@ -1,56 +1,65 @@
+0BSD
 AAL
-Abstyles
-Adobe-2006
-Adobe-Glyph
 ADSL
 AFL-1.1
 AFL-1.2
 AFL-2.0
 AFL-2.1
 AFL-3.0
-Afmparse
 AGPL-1.0
+AGPL-1.0-only
+AGPL-1.0-or-later
 AGPL-3.0
-Aladdin
+AGPL-3.0-only
+AGPL-3.0-or-later
 AMDPLPA
 AML
 AMPAS
 ANTLR-PD
-Apache-1.0
-Apache-1.1
-Apache-2.0
 APAFML
 APL-1.0
 APSL-1.0
 APSL-1.1
 APSL-1.2
 APSL-2.0
+Abstyles
+Adobe-2006
+Adobe-Glyph
+Afmparse
+Aladdin
+Apache-1.0
+Apache-1.1
+Apache-2.0
 Artistic-1.0
-Artistic-1.0-cl8
 Artistic-1.0-Perl
+Artistic-1.0-cl8
 Artistic-2.0
+BSD-1-Clause
+BSD-2-Clause
+BSD-2-Clause-FreeBSD
+BSD-2-Clause-NetBSD
+BSD-2-Clause-Patent
+BSD-3-Clause
+BSD-3-Clause-Attribution
+BSD-3-Clause-Clear
+BSD-3-Clause-LBNL
+BSD-3-Clause-No-Nuclear-License
+BSD-3-Clause-No-Nuclear-License-2014
+BSD-3-Clause-No-Nuclear-Warranty
+BSD-3-Clause-Open-MPI
+BSD-4-Clause
+BSD-4-Clause-UC
+BSD-Protection
+BSD-Source-Code
+BSL-1.0
 Bahyph
 Barr
 Beerware
 BitTorrent-1.0
 BitTorrent-1.1
+BlueOak-1.0.0
 Borceux
-BSD-2-Clause
-BSD-2-Clause-FreeBSD
-BSD-2-Clause-NetBSD
-BSD-3-Clause
-BSD-3-Clause-Attribution
-BSD-3-Clause-Clear
-BSD-3-Clause-LBNL
-BSD-4-Clause
-BSD-4-Clause-UC
-BSD-Protection
-BSL-1.0
-bzip2-1.0.5
-bzip2-1.0.6
-Caldera
 CATOSL-1.1
-CC0-1.0
 CC-BY-1.0
 CC-BY-2.0
 CC-BY-2.5
@@ -81,96 +90,122 @@ CC-BY-SA-2.0
 CC-BY-SA-2.5
 CC-BY-SA-3.0
 CC-BY-SA-4.0
+CC-PDDC
+CC0-1.0
 CDDL-1.0
 CDDL-1.1
+CDLA-Permissive-1.0
+CDLA-Sharing-1.0
 CECILL-1.0
 CECILL-1.1
 CECILL-2.0
+CECILL-2.1
 CECILL-B
 CECILL-C
-ClArtistic
+CERN-OHL-1.1
+CERN-OHL-1.2
+CNRI-Jython
 CNRI-Python
 CNRI-Python-GPL-Compatible
-Condor-1.1
 CPAL-1.0
 CPL-1.0
 CPOL-1.02
-Crossword
 CUA-OPL-1.0
+Caldera
+ClArtistic
+Condor-1.1
+Crossword
+CrystalStacker
 Cube
 D-FSL-1.0
-diffmark
 DOC
-Dotseqn
 DSDP
-dvipdfm
+Dotseqn
 ECL-1.0
 ECL-2.0
-eCos-2.0
 EFL-1.0
 EFL-2.0
-eGenix
-Entessa
 EPL-1.0
-ErlPL-1.1
+EPL-2.0
 EUDatagrid
 EUPL-1.0
 EUPL-1.1
+EUPL-1.2
+Entessa
+ErlPL-1.1
 Eurosym
-Fair
-Frameworx-1.0
-FreeImage
+FSFAP
 FSFUL
 FSFULLR
 FTL
+Fair
+Frameworx-1.0
+FreeImage
 GFDL-1.1
+GFDL-1.1-only
+GFDL-1.1-or-later
 GFDL-1.2
+GFDL-1.2-only
+GFDL-1.2-or-later
 GFDL-1.3
-GFDL-1.3+
-Giftware
+GFDL-1.3-only
+GFDL-1.3-or-later
 GL2PS
-Glide
-Glulxe
-gnuplot
 GPL-1.0
 GPL-1.0+
+GPL-1.0-only
+GPL-1.0-or-later
 GPL-2.0
 GPL-2.0+
+GPL-2.0-only
+GPL-2.0-or-later
+GPL-2.0-with-GCC-exception
 GPL-2.0-with-autoconf-exception
 GPL-2.0-with-bison-exception
 GPL-2.0-with-classpath-exception
 GPL-2.0-with-font-exception
-GPL-2.0-with-GCC-exception
 GPL-3.0
 GPL-3.0+
-GPL-3.0-with-autoconf-exception
+GPL-3.0-only
+GPL-3.0-or-later
 GPL-3.0-with-GCC-exception
-gSOAP-1.3b
-HaskellReport
+GPL-3.0-with-autoconf-exception
+Giftware
+Glide
+Glulxe
 HPND
+HPND-sell-variant
+HaskellReport
 IBM-pibs
 ICU
 IJG
-ImageMagick
-iMatix
-Imlib2
-Intel
-Intel-ACPI
 IPA
 IPL-1.0
 ISC
-JasPer-2.0
+ImageMagick
+Imlib2
+Info-ZIP
+Intel
+Intel-ACPI
+Interbase-1.0
+JPNIC
 JSON
-Latex2e
-Leptonica
+JasPer-2.0
+LAL-1.2
+LAL-1.3
 LGPL-2.0
 LGPL-2.0+
+LGPL-2.0-only
+LGPL-2.0-or-later
 LGPL-2.1
 LGPL-2.1+
+LGPL-2.1-only
+LGPL-2.1-or-later
 LGPL-3.0
 LGPL-3.0+
-Libpng
-libtiff
+LGPL-3.0-only
+LGPL-3.0-or-later
+LGPLLR
 LPL-1.0
 LPL-1.02
 LPPL-1.0
@@ -178,17 +213,20 @@ LPPL-1.1
 LPPL-1.2
 LPPL-1.3a
 LPPL-1.3c
-MakeIndex
-MirOS
+Latex2e
+Leptonica
+LiLiQ-P-1.1
+LiLiQ-R-1.1
+LiLiQ-Rplus-1.1
+Libpng
+Linux-OpenIB
 MIT
-MIT-advertising
+MIT-0
 MIT-CMU
+MIT-advertising
 MIT-enna
 MIT-feh
-MIT-Opengroup
 MITNFA
-Motosoto
-mpich2
 MPL-1.0
 MPL-1.1
 MPL-2.0
@@ -196,29 +234,46 @@ MPL-2.0-no-copyleft-exception
 MS-PL
 MS-RL
 MTLL
+MakeIndex
+MirOS
+Motosoto
+MulanPSL-1.0
 Multics
 Mup
 NASA-1.3
-Naumen
 NBPL-1.0
 NCSA
-NetCDF
-Newsletr
 NGPL
+NLOD-1.0
 NLPL
-Nokia
 NOSL
-Noweb
 NPL-1.0
 NPL-1.1
 NPOSL-3.0
 NRL
 NTP
+NTP-0
+Naumen
+Net-SNMP
+NetCDF
+Newsletr
+Nokia
+Noweb
 Nunit
+OCCT-PL
 OCLC-2.0
+ODC-By-1.0
 ODbL-1.0
 OFL-1.0
+OFL-1.0-RFN
+OFL-1.0-no-RFN
 OFL-1.1
+OFL-1.1-RFN
+OFL-1.1-no-RFN
+OGL-Canada-2.0
+OGL-UK-1.0
+OGL-UK-2.0
+OGL-UK-3.0
 OGTSL
 OLDAP-1.1
 OLDAP-1.2
@@ -237,77 +292,118 @@ OLDAP-2.6
 OLDAP-2.7
 OLDAP-2.8
 OML
-OpenSSL
 OPL-1.0
+OSET-PL-2.1
 OSL-1.0
 OSL-1.1
 OSL-2.0
 OSL-2.1
 OSL-3.0
+OpenSSL
 PDDL-1.0
 PHP-3.0
 PHP-3.01
+PSF-2.0
+Parity-6.0.0
 Plexus
 PostgreSQL
-psfrag
-psutils
 Python-2.0
-Qhull
 QPL-1.0
-Rdisc
+Qhull
 RHeCos-1.1
 RPL-1.1
 RPL-1.5
 RPSL-1.0
+RSA-MD
 RSCPL
+Rdisc
 Ruby
-Saxpath
 SAX-PD
 SCEA
 SGI-B-1.0
 SGI-B-1.1
 SGI-B-2.0
-SimPL-2.0
+SHL-0.5
+SHL-0.51
 SISSL
 SISSL-1.2
-Sleepycat
 SMLNJ
+SMPPL
 SNIA
 SPL-1.0
+SSH-OpenSSH
+SSH-short
+SSPL-1.0
+SWL
+Saxpath
+Sendmail
+Sendmail-8.23
+SimPL-2.0
+Sleepycat
+Spencer-86
+Spencer-94
+Spencer-99
 StandardML-NJ
 SugarCRM-1.1.3
-SWL
+TAPR-OHL-1.0
 TCL
+TCP-wrappers
 TMate
 TORQUE-1.1
 TOSL
+TU-Berlin-1.0
+TU-Berlin-2.0
+UCL-1.0
+UPL-1.0
+Unicode-DFS-2015
+Unicode-DFS-2016
 Unicode-TOU
 Unlicense
-Vim
 VOSTROM
 VSL-1.0
+Vim
 W3C
 W3C-19980720
+W3C-20150513
+WTFPL
 Watcom-1.0
 Wsuipa
-WTFPL
-WXwindows
 X11
-Xerox
 XFree86-1.1
-xinetd
-Xnet
-xpp
 XSkat
+Xerox
+Xnet
 YPL-1.0
 YPL-1.1
+ZPL-1.1
+ZPL-2.0
+ZPL-2.1
 Zed
 Zend-2.0
 Zimbra-1.3
 Zimbra-1.4
 Zlib
+blessing
+bzip2-1.0.5
+bzip2-1.0.6
+copyleft-next-0.3.0
+copyleft-next-0.3.1
+curl
+diffmark
+dvipdfm
+eCos-2.0
+eGenix
+etalab-2.0
+gSOAP-1.3b
+gnuplot
+iMatix
+libpng-2.0
+libselinux-1.0
+libtiff
+mpich2
+psfrag
+psutils
+wxWindows
+xinetd
+xpp
 zlib-acknowledgement
-ZPL-1.1
-ZPL-2.0
-ZPL-2.1
-w3c

lts/Makefile

@@ -0,0 +1,5 @@
+TEST = test.test_ltsutils
+
+.PHONY: test
+test:
+	PYTHONPATH=. python -m unittest -v -k .Test $(TEST)

lts/README.md

@@ -0,0 +1,19 @@
+# LTS package maintenance utility
+
+This tooling is designed to automate 2 main tasks that are part of
+the package maintenance workflow of Clear Linux LTS. These tasks are:
+- Back-porting of a patch (e.g. security fix) to older branches.
+- Building RPMs with the intent of sharing binaries of older LTS branches to
+  newer branches whenever possible.
+
+There should be no need to run this tool directly. Instead use the following
+targets defined in Makefile.common.lts:
+- lts-show: Show a summary of active LTS branches
+- lts-backport: Attempt to fast-forward the previous active branch to the current branch
+- lts-build: Build RPM in Koji, or reuse existing build from older branch
+
+"Active" branches correspond to LTS releases that currently have support.
+They are listed in a flat file "active-branches" in "lts" directory, from
+oldest to newest. New entries are added by Clear Linux LTS developers as
+new releases become available, and entries removed as releases become
+obsolete.

lts/ltsutils/package_repo.py

@@ -0,0 +1,52 @@
+import pathlib
+from subprocess import PIPE, CalledProcessError
+from .shell import Shell
+
+class PackageRepo:
+    '''Represents a package repository. Most methods are wrappers of git commands.'''
+    class UnknownCurrentBranchException(Exception): pass
+    class InvalidBranchException(Exception): pass
+
+    def __init__(self, name, path):
+        self.name = name
+        self.path = pathlib.Path(path)
+        self.sh = Shell(self.path)
+
+    def getNVR(self, commit='HEAD'):
+        with self.sh.popen(['git', 'show', '{}:{}.spec'.format(commit, self.name)], stdout=PIPE) as specfile:
+            nvr = self.sh.run('rpmspec --srpm -q --queryformat %{NVR} /dev/stdin', stdin=specfile.stdout)
+        return tuple(nvr.stdout.strip().rsplit('-', maxsplit=2))
+
+    def checkoutBranch(self, branch, allow_remote=False):
+        # allow_remote=True allows checking out a new remote-tracking branch
+        if not allow_remote and not self.hasBranch(branch):
+            raise self.InvalidBranchException(branch)
+        self.sh.run_args(['git', 'checkout', branch], capture_output=False)
+
+    def fastForwardBranch(self, old, new):
+        self.checkoutBranch(old)
+        if not self.hasBranch(new):
+            raise self.InvalidBranchException(new)
+        self.sh.run_args(['git', 'merge', '--ff-only', new], capture_output=False)
+
+    def getActiveBranches(self):
+        toplvl = pathlib.Path(self.path) / '../..'
+        common = toplvl / 'projects/common'
+        active_branches = common / 'lts/active-branches'
+        with active_branches.open() as f:
+            return [line.rstrip() for line in f]
+
+    def getCurrentBranch(self):
+        try:
+            head = self.sh.run('git symbolic-ref HEAD').stdout.strip()
+        except CalledProcessError:
+            raise self.UnknownCurrentBranchException
+
+        refs_heads = 'refs/heads/'
+        assert head.startswith(refs_heads)
+        head = head[len(refs_heads):]
+        return head
+
+    def hasBranch(self, branch):
+        p = self.sh.run_args(['git', 'rev-parse', 'refs/heads/'+branch], check=False)
+        return p.returncode == 0

lts/ltsutils/shell.py

@@ -0,0 +1,27 @@
+import subprocess, shlex
+
+class Shell:
+    # Default options passed to subprocess.run. May be customized per-instance.
+    cwd = None
+    check = True
+    capture_output = True
+    text = True
+
+    def __init__(self, cwd=None):
+        if cwd: self.cwd = cwd
+
+    def run_args(self, args, **kwargs):
+        kwargs1 = {
+                'check': self.check,
+                'capture_output': self.capture_output,
+                'text': self.text,
+                'cwd': self.cwd
+                }
+        kwargs1.update(kwargs)
+        return subprocess.run(args, **kwargs1)
+
+    def run(self, cmd, **kwargs):
+        return self.run_args(shlex.split(cmd), **kwargs)
+
+    def popen(self, args, **kwargs):
+        return subprocess.Popen(args, cwd=self.cwd, **kwargs)

lts/main.py

@@ -0,0 +1,161 @@
+#!/usr/bin/python
+import sys,argparse
+
+from ltsutils.package_repo import PackageRepo
+
+def log(msg, **kwargs):
+    print(msg, file=sys.stderr)
+
+def init_parser():
+    main = argparse.ArgumentParser()
+    main.add_argument('package_name', nargs=1)
+    subparsers = main.add_subparsers(dest='command', metavar='command', required=True)
+
+    # Package maintenance commands
+    p = subparsers.add_parser('prev-branch',
+            help='show previous branch')
+    p.add_argument('--checkout', action='store_true', help='checkout the branch')
+    p = subparsers.add_parser('next-branch',
+            help='show next branch')
+    p.add_argument('--checkout', action='store_true', help='checkout the branch')
+    p = subparsers.add_parser('current-branch',
+            help='show current branch')
+    p = subparsers.add_parser('is-same-version',
+            help='return true if package version is the same as the given branch')
+    p.add_argument('branch', nargs=1)
+    p = subparsers.add_parser('fast-forward',
+            help='fast-forward current branch to a newer branch')
+    p.add_argument('branch', nargs=1)
+
+    # RPM build commands
+    p = subparsers.add_parser('can-reuse-binary',
+            help='check if binary from another branch can be used in current branch')
+    p.add_argument('branch', nargs=1)
+
+    # Other commands
+    p = subparsers.add_parser('prompt',
+            help='prompt user to continue and return appropriate exit code')
+    p = subparsers.add_parser('sanity-check',
+            help='run sanity checks for data consistency')
+
+    return main
+
+def prev_branch(args, repo):
+    active_branches = repo.getActiveBranches()
+    current = repo.getCurrentBranch()
+
+    i = active_branches.index(current)
+    if i == 0:
+        log('Already on oldest active branch.')
+        return False
+
+    prev = active_branches[i-1]
+    print(prev)
+    if args.checkout:
+        repo.checkoutBranch(prev, allow_remote=True)
+
+def next_branch(args, repo):
+    active_branches = repo.getActiveBranches()
+    current = repo.getCurrentBranch()
+
+    i = active_branches.index(current)
+    if i == len(active_branches)-1:
+        log('Already on newest active branch.')
+        return False
+
+    next_ = active_branches[i+1]
+    print(next_)
+    if args.checkout:
+        repo.checkoutBranch(next_, allow_remote=False)
+
+def current_branch(args, repo):
+    current = repo.getCurrentBranch()
+    print(current)
+
+def is_same_version(args, other):
+    current = repo.getCurrentBranch()
+    other = args.branch[0]
+    assert repo.hasBranch(other), 'Branch %s not found' % other
+
+    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (current, other)]
+    if v1 != v2:
+        log('Current version {} does not match version {} on branch {}'.format(v1, v2, other))
+    return v1 == v2
+
+def fast_forward(args, repo):
+    current = repo.getCurrentBranch()
+    newer = args.branch[0]
+
+    log('Fast-forwarding {} to {}'.format(current, newer))
+    repo.fastForwardBranch(current, newer)
+
+def sanity_check(args, repo):
+    ok = True
+    # HEAD must point to a branch
+    try:
+        current = repo.getCurrentBranch()
+    except PackageRepo.UnknownCurrentBranchException:
+        log('Unknown current branch. Has a branch been checked out?')
+        current = None
+        ok = False
+
+    # active-branches file must not be empty
+    active_branches = repo.getActiveBranches()
+    if not len(active_branches):
+        log('No active branches defined. Is active-branches file empty?')
+        ok = False
+    # current branch must be an active branch
+    elif current and current not in active_branches:
+        log('%s is not an active LTS branch.' % current)
+        ok = False
+
+    return ok
+
+def can_reuse_binary(args, repo):
+    # Just compare versions for now
+    # TODO: ABI compatibility testing
+    current = repo.getCurrentBranch()
+    older = args.branch[0]
+    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (older, current)]
+    return v1 == v2
+
+def prompt(args, repo):
+    import selectors
+    timeout = 60
+
+    while True:
+        print('Continue? (y/N): ', end='', flush=True)
+        with selectors.DefaultSelector() as sel:
+            sel.register(sys.stdin, selectors.EVENT_READ)
+            events = sel.select(timeout)
+        if not len(events):
+            print('Timed out after {}s.'.format(timeout))
+            return False
+        else:
+            s = sys.stdin.readline().rstrip('\n')
+            if s in ('Y', 'y', 'N', 'n', ''):
+                break
+
+    if s in ('Y', 'y'):
+        return True
+    else:
+        print('Cancelled.')
+        return False
+
+if __name__=='__main__':
+    args = init_parser().parse_args()
+    repo = PackageRepo(args.package_name[0], '.')
+
+    commands = {
+            'prev-branch': prev_branch,
+            'next-branch': next_branch,
+            'current-branch': current_branch,
+            'is-same-version': is_same_version,
+            'fast-forward': fast_forward,
+            'can-reuse-binary': can_reuse_binary,
+            'prompt': prompt,
+            'sanity-check': sanity_check,
+            }
+    ret = commands[args.command](args, repo)
+    if ret is not None:
+        exit(0 if ret else 1)

lts/test/test_ltsutils.py

@@ -0,0 +1,97 @@
+import unittest
+import os, pathlib, tempfile, logging
+import subprocess, functools
+
+from ltsutils.package_repo import PackageRepo
+import ltsutils.shell
+
+run = functools.partial(subprocess.run, check=True)
+
+class Package:
+    def __init__(self, name):
+        self.name = name
+        self.repo_url = 'https://github.com/clearlinux-pkgs/{}.git'.format(name)
+
+class LTSUtilsTestCase(unittest.TestCase):
+    toplvl = pathlib.Path('../../..')
+    packages = toplvl / 'packages'
+
+    def cloneOrExtractRepo(self):
+        tmpdir = pathlib.Path('/var/tmp/common-lts-test')
+        tmpdir.mkdir(mode=0o700, exist_ok=True)
+        tarball = tmpdir / '{}.tar.gz'.format(self.package.name)
+        if tarball.exists():
+            run(['tar', 'xf', tarball, '-C', self.workdir])
+        else:
+            run(['git', 'clone', self.package.repo_url, self.workdir])
+            run(['tar', 'czf', tarball, '-C', self.workdir, '.'])
+
+    def setUp(self):
+        self._tmpdir = tempfile.TemporaryDirectory(prefix='test-{}-'.format(self.package.name), dir=self.packages)
+        self.workdir = pathlib.Path(self._tmpdir.name)
+        self.cloneOrExtractRepo()
+        self.repo = PackageRepo(self.package.name, self.workdir)
+
+        self._sh = ltsutils.shell.Shell(self.workdir)
+        self._sh.capture_output = False
+
+    def sh(self, cmd):
+        return self._sh.run(cmd)
+
+    def sh_stdout(self, cmd):
+        return self._sh.run(cmd, capture_output=True).stdout.rstrip()
+
+    def tearDown(self):
+        self._tmpdir.cleanup()
+
+class PackageRepoTestCase(LTSUtilsTestCase):
+    def setUp(self):
+        super().setUp()
+        self.L1 = self.package.L1
+        self.L2 = self.package.L2
+        self.sh('git branch L1 %s' % self.L1)
+        self.sh('git branch L2 %s' % self.L2)
+
+    def testGetNVR(self):
+        raise NotImplementedError
+
+    def testHasBranch(self):
+        self.assertTrue(self.repo.hasBranch('L2'))
+        self.assertFalse(self.repo.hasBranch('L3'))
+
+    def testCheckoutBranch(self):
+        self.repo.checkoutBranch('L2')
+        self.assertEqual(self.sh_stdout('git rev-parse HEAD'), self.L2)
+
+        self.assertRaises(PackageRepo.InvalidBranchException, self.repo.checkoutBranch, 'L3')
+
+    def testFastForward(self):
+        self.repo.fastForwardBranch('L1', 'L2')
+        self.assertEqual(self.sh_stdout('git rev-parse L1'), self.L2)
+        self.assertEqual(self.sh_stdout('git rev-parse L2'), self.L2)
+
+    def testGetCurrentBranch(self):
+        self.repo.checkoutBranch('L2')
+        b = self.repo.getCurrentBranch()
+        self.assertEqual(b, 'L2')
+
+        self.sh('git checkout --detach L2')
+        self.assertRaises(PackageRepo.UnknownCurrentBranchException, self.repo.getCurrentBranch)
+
+class TestNano(PackageRepoTestCase):
+    package = Package('nano')
+    package.L1 = '3dcfa09f5217eedf6ec7539af7e243655d3abdb6' # 3.2-54
+    package.L2 = 'b8243dd54e8feb16a11474f848b8735f5591cf12' # 3.2-55
+
+    def testGetNVR(self):
+        nvr = self.repo.getNVR(self.L2)
+        self.assertEqual(nvr, ('nano', '3.2', '55'))
+
+class TestMySQL_Python(PackageRepoTestCase):
+    package = Package('MySQL-python')
+    package.L1 = '386163d8fc9c857c7194c4e958374af4c4f071ed' # 1.2.5-31
+    package.L2 = 'f85bc5ec2141384f45f224d4464a0a44a981a4d4' # 1.2.5-33
+
+    def testGetNVR(self):
+        nvr = self.repo.getNVR(self.L2)
+        self.assertEqual(nvr, ('MySQL-python', '1.2.5', '33'))

mapping

@@ -51,4 +51,5 @@ rack = rubygem-rack
 compute = nova
 international_components_for_unicode = icu4c
 node.js = nodejs
-jupyter-core = jupyter_core
+jupyter-core = jupyter_core
+flatten-json = flatten_json

patchfilter.py

@@ -0,0 +1,393 @@
+#!/usr/bin/python3
+import sys
+import tempfile
+import subprocess
+
+header = list()
+files = list()
+files_chunks = dict()	# dict to list
+files_header = dict()	# dict to list
+
+
+
+def push_chunk(file, chunk):
+    global files_chunks
+    
+    if len(chunk) == 0:
+        return
+    
+    if file not in files_chunks:
+        files_chunks[file] = list()
+        
+    files_chunks[file].append(chunk)
+
+
+def parse_patch(lines):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    
+    phase = 0
+    
+    currentfile = ""
+    currentchunk = list()
+    
+        
+    for line in lines:
+        line = line.replace("\n","")
+        
+        if line.startswith("diff --git"):
+        
+            push_chunk(currentfile, currentchunk)
+            currentchunk = list()
+        
+            filename = line.replace("diff --git","")
+            index= filename.find("b/")
+            if index >= 0:
+                filename = filename[index:].strip()
+                currentfile = filename
+                files.append(currentfile)
+                phase = 1
+                
+        if line.startswith("@@"):
+            push_chunk(currentfile, currentchunk)
+            currentchunk = list()
+            phase = 2
+            
+            
+        if phase == 0:
+            header.append(line)
+        if phase == 1:
+            if currentfile not in files_header:
+                files_header[currentfile] = list()
+            files_header[currentfile].append(line) 
+        if phase == 2:
+            currentchunk.append(line)
+        
+    push_chunk(currentfile, currentchunk)
+    currentchunk = list()
+
+def print_all():
+    global header
+    global files
+    global files_chunks
+    global files_header
+
+    for line in header:
+        print(line)
+    for file in files:
+        for line in files_header[file]:
+            print(line)
+        if file in files_chunks:
+            for chunk in files_chunks[file]:
+                for line in chunk:
+                    print(line)
+                    
+                    
+def zap_entire_file(filename):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    if filename in files:
+        files.remove(filename)                
+
+def zap_entire_file_end(filename):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            files.remove(file)                
+            
+def zap_line_in_file_substring(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_delete = list()
+                for line in chunk:
+                    if match in line:
+                        to_delete.append(line)
+                for line in to_delete:
+                    chunk.remove(line)
+            
+
+def zap_line_in_file(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[filename]:
+                to_delete = list()
+                for line in chunk:
+                    if line == ("+" + match):
+                        to_delete.append(line)
+                    if line == ("-" + match):
+                        to_delete.append(line)
+                for line in to_delete:
+                    chunk.remove(line)
+            
+
+def zap_line_in_file_start(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_remove = list()
+                for line in chunk:
+                    if line.startswith("+" + match):
+                        to_remove.append(line)
+                    if line.startswith("-" + match):
+                        to_remove.append(line)
+                for line in to_remove:
+                    chunk.remove(line)
+            
+def zap_plus_line_in_file(filename):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_remove = list()
+                for line in chunk:
+                    if line.startswith("+"):
+                        to_remove.append(line)
+                for line in to_remove:
+                    chunk.remove(line)
+            
+
+
+def chunk_empty(chunk):
+    for line in chunk:
+        if len(line) > 0 and line[0] == '+':
+            return False
+    return True
+                           
+def zap_empty_chunks():
+    global header
+    global files
+    global files_chunks
+    global files_header
+    files_to_remove = list()
+    for file in files:
+        if file not in files_chunks:
+            files.remove(file)
+            continue
+        to_remove = list()
+        for chunk in files_chunks[file]:
+            if chunk_empty(chunk):
+                to_remove.append(chunk)
+        for c in to_remove:
+            files_chunks[file].remove(c)
+        if len(files_chunks[file]) == 0:
+            files_to_remove.append(file)
+            
+    for file in files_to_remove:
+        files.remove(file)
+            
+            
+
+
+def main():
+
+    if len(sys.argv) > 1:
+        filename = sys.argv[1]
+    
+        with open (filename, "r") as myfile:
+            lines = myfile.readlines()
+    else:
+        output = subprocess.check_output("git format-patch -1 --stdout", shell=True).decode("latin-1")
+        lines = output.split("\n")
+        
+    parse_patch(lines)
+
+    zap_entire_file("b/release")
+    zap_entire_file("b/buildreq_cache")
+    zap_entire_file("b/.gitignore")
+    zap_entire_file("b/upstream")
+    zap_entire_file("b/NEWS")
+    zap_entire_file("b/ChangeLog")
+    zap_entire_file_end("xz.sig")
+    zap_entire_file_end("gz.sig")
+    zap_entire_file_end("bz2.sig")
+    zap_entire_file_end("xz.asc")
+    zap_entire_file_end("gz.asc")
+    zap_entire_file_end("bz2.asc")
+    
+    zap_entire_file_end(".pkey")
+    
+    zap_line_in_file("b/testresults", "Total : 0")
+    zap_line_in_file("b/testresults", "Pass : 0")
+    zap_line_in_file("b/testresults", "Fail : 0")
+    zap_line_in_file("b/testresults", "XFail : 0")
+    zap_line_in_file("b/testresults", "Skip : 0")
+    
+    zap_line_in_file("b/requires_ban", "#FOO")
+    zap_line_in_file("b/Makefile", "ARCHIVES = ")
+    zap_line_in_file("b/Makefile", "include ../common/Makefile.common")
+    zap_line_in_file("b/Makefile", "")
+    
+    zap_line_in_file_start(".spec", "Release  :")
+    zap_line_in_file_start(".spec", "Source99 :")
+    zap_line_in_file_start(".spec", "Source0  :")
+    zap_line_in_file_start(".spec", "export SOURCE_DATE_EPOCH")
+    zap_line_in_file_start(".spec", "export AR=gcc-ar")
+    zap_line_in_file_start(".spec", "export RANLIB=gcc-ranlib")
+    zap_line_in_file_start(".spec", "export NM=gcc-nm")
+    zap_line_in_file_start(".spec", "export CFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FCFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export CXXFLAGS=\"$CXXFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "%setup -q -n ")
+    zap_line_in_file_start(".spec", "URL      :")
+    zap_line_in_file_start(".spec", "Version  :")
+    zap_line_in_file_start(".spec", "Group    : Development/Tools")
+    zap_line_in_file_start(".spec", "No detailed description available")
+    zap_line_in_file_start(".spec", "bin components for the")
+    zap_line_in_file_start(".spec", "dev components for the")
+    zap_line_in_file_start(".spec", "lib components for the")
+    zap_line_in_file_start(".spec", "data components for the")
+    zap_line_in_file_start(".spec", "locales components for the")
+    zap_line_in_file_start(".spec", "license components for the")
+    zap_line_in_file_start(".spec", "doc components for the")
+    zap_line_in_file_start(".spec", "Group: Default")
+    zap_line_in_file_start(".spec", "export http_proxy=http://127.0.0.1:9/")
+    zap_line_in_file_start(".spec", "export https_proxy=http://127.0.0.1:9/")
+    zap_line_in_file_start(".spec", "export ftp_proxy=http://127.0.0.1:9/")
+    zap_line_in_file_start(".spec", "export no_proxy=localhost,127.0.0.1,0.0.0.0")
+    zap_line_in_file_start(".spec", "Summary:")
+    zap_line_in_file_start(".spec", "Group: Development")
+    zap_line_in_file_start(".spec", "Group: Binaries")
+    zap_line_in_file_start(".spec", "Group: Data")
+    zap_line_in_file_start(".spec", "Group: Libraries")
+    zap_line_in_file_start(".spec", "Group: Documentation")
+    zap_line_in_file_start(".spec", "export LANG=C")
+    zap_line_in_file_start(".spec", "%description lib")
+    zap_line_in_file_start(".spec", "%description bin")
+    zap_line_in_file_start(".spec", "%description data")
+    zap_line_in_file_start(".spec", "%description doc")
+    zap_line_in_file_start(".spec", "%description locales")
+    zap_line_in_file_start(".spec", "%description dev")
+    zap_line_in_file_start(".spec", "%description license")
+    zap_line_in_file_start(".spec", "%package doc")
+    zap_line_in_file_start(".spec", "cp -a ")
+    zap_line_in_file_start(".spec", "cd %{_builddir}/")
+    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
+ 
+    zap_line_in_file_start("b/Makefile", "URL =")
+    zap_line_in_file_start("b/Makefile", "PKG_NAME :=")
+
+    zap_line_in_file("b/options.conf", "archives = ")
+    zap_line_in_file("b/options.conf", "giturl = ")
+    zap_line_in_file("b/options.conf", "url = ")
+    zap_line_in_file("b/options.conf", "[package]")
+    zap_line_in_file("b/options.conf", "")
+    zap_line_in_file("b/options.conf", "[autospec]")
+    zap_line_in_file("b/options.conf", "# build 32 bit libraries")
+    zap_line_in_file("b/options.conf", "32bit = false")
+    zap_line_in_file("b/options.conf", "# allow package to build with test failures")
+    zap_line_in_file("b/options.conf", "allow_test_failures = false")
+    zap_line_in_file("b/options.conf", "# unset %build ld_as_needed variable")
+    zap_line_in_file("b/options.conf", "asneeded = false")
+    zap_line_in_file("b/options.conf", "# this package is trusted enough to automatically update (used by other tools)")
+    zap_line_in_file("b/options.conf", "autoupdate = false")
+    zap_line_in_file("b/options.conf", "# extend flags with '-std=gnu++98")
+    zap_line_in_file("b/options.conf", "broken_c++ = false")
+    zap_line_in_file("b/options.conf", "# disable parallelization during build")
+    zap_line_in_file("b/options.conf", "broken_parallel_build = false")
+    zap_line_in_file("b/options.conf", "# this package is a library compatability package and only ships versioned library files")
+    zap_line_in_file("b/options.conf", "# this package is a library compatibility package and only ships versioned library files")
+    zap_line_in_file("b/options.conf", "compat = false")
+    zap_line_in_file("b/options.conf", "# set conservative build flags")
+    zap_line_in_file("b/options.conf", "conservative_flags = false")
+    zap_line_in_file("b/options.conf", "# dev package requires the extras to be installed")
+    zap_line_in_file("b/options.conf", "dev_requires_extras = false")
+    zap_line_in_file("b/options.conf", "nodebug = false")
+    zap_line_in_file("b/options.conf", "# pass -ffast-math to compiler")
+    zap_line_in_file("b/options.conf", "fast-math = false")
+    zap_line_in_file("b/options.conf", "# optimize build for speed over size")
+    zap_line_in_file("b/options.conf", "funroll-loops = false")
+    zap_line_in_file("b/options.conf", "# set flags to smallest -02 flags possible")
+    zap_line_in_file("b/options.conf", "insecure_build = false")
+    zap_line_in_file("b/options.conf", "# do not remove static libraries")
+    zap_line_in_file("b/options.conf", "keepstatic = false")
+    zap_line_in_file("b/options.conf", "# do not require autostart subpackage")
+    zap_line_in_file("b/options.conf", "no_autostart = false")
+    zap_line_in_file("b/options.conf", "# disable stripping binaries")
+    zap_line_in_file("b/options.conf", "nostrip = false")
+    zap_line_in_file("b/options.conf", "# optimize build for size over speed")
+    zap_line_in_file("b/options.conf", "optimize_size = false")
+    zap_line_in_file("b/options.conf", "# set profile for pgo")
+    zap_line_in_file("b/options.conf", "pgo = false")
+    zap_line_in_file("b/options.conf", "# set flags for security-sensitive builds")
+    zap_line_in_file("b/options.conf", "security_sensitive = false")
+    zap_line_in_file("b/options.conf", "# do not run test suite")
+    zap_line_in_file("b/options.conf", "skip_tests = false")
+    zap_line_in_file("b/options.conf", "# add .so files to the lib package instead of dev")
+    zap_line_in_file("b/options.conf", "so_to_lib = false")
+    zap_line_in_file("b/options.conf", "# configure build for avx2")
+    zap_line_in_file("b/options.conf", "use_avx2 = false")
+    zap_line_in_file("b/options.conf", "# configure build for avx512")
+    zap_line_in_file("b/options.conf", "use_avx512 = false")
+    zap_line_in_file("b/options.conf", "# add clang flags")
+    zap_line_in_file("b/options.conf", "use_clang = false")
+    zap_line_in_file("b/options.conf", "# configure build for lto")
+    zap_line_in_file("b/options.conf", "use_lto = true")
+    zap_line_in_file("b/options.conf", "# require package verification for build")
+    zap_line_in_file("b/options.conf", "verify_required = true")
+    zap_line_in_file("b/options.conf", "# do not generate debuginfo for this package")
+    zap_line_in_file("b/options.conf", "nodebug = false")
+    zap_line_in_file("b/options.conf", "domain =")
+
+    zap_line_in_file("b/buildreq_add", "# This file contains additional build requirements that did not get")
+    zap_line_in_file("b/buildreq_add", "# picked up automatically. One name per line, no whitespace.")
+    zap_line_in_file("b/buildreq_ban", "# This file contains build requirements that get picked up but are")
+    zap_line_in_file("b/buildreq_ban", "# undesirable. One entry per line, no whitespace.")
+    
+    
+    zap_line_in_file("b/excludes", "# This file contains the output files that need %exclude. Full path")
+    zap_line_in_file("b/excludes", "# names, one per line.")
+
+    zap_line_in_file("b/pkgconfig_add", "# This file contains additional pkgconfig build requirements that did")
+    zap_line_in_file("b/pkgconfig_add", "# not get picked up automatically. One name per line, no whitespace.")
+    zap_line_in_file("b/pkgconfig_ban", "# This file contains pkgconfig build requirements that get picked up")
+    zap_line_in_file("b/pkgconfig_ban", "# but are undesirable. One entry per line, no whitespace.")
+    zap_line_in_file("b/requires_add", "# This file contains additional runtime requirements that did not get")
+    zap_line_in_file("b/requires_add", "# picked up automatically. One name per line, no whitespace.")
+    zap_line_in_file("b/requires_ban", "# This file contains runtime requirements that get picked up but are")
+    zap_line_in_file("b/requires_ban", "# undesirable. One entry per line, no whitespace.")
+    
+    zap_line_in_file_start(".spec", "Summary  : No detailed summary available")
+    
+    zap_plus_line_in_file("symbols")
+    zap_plus_line_in_file("symbols32")
+
+    zap_entire_file("b/whatrequires")
+    zap_entire_file("b/versions")
+    zap_entire_file("b/.gitignore")
+    
+
+    zap_line_in_file_substring(".spec", "%{buildroot}/usr/share/package-licenses")   # version number change in license copy
+    zap_empty_chunks()
+    
+    print_all()
+
+if __name__ == '__main__':
+    with tempfile.TemporaryDirectory() as workingdir:
+        main()
+

provides.sh

@@ -23,7 +23,7 @@ if [ "$#" -eq 0 ]; then
 fi
 
 if [ ! -e /etc/yum.conf ]; then
-  echo "Error: yum.conf is missing. Please copy projects/common/image-creator/yum.conf to /etc"
+  echo "Error: yum.conf is missing. Please copy projects/common/conf/yum.conf to /etc"
   exit 1
 fi
 

start_qemu.sh

@@ -43,15 +43,24 @@ if [ ! -f "$IMAGE" ]; then
 fi
 rm -f debug.log
 
+# 10/25/2018: keep back compatibility for a while
+UEFI_BIOS="-bios OVMF.fd"
+
+if [ -f OVMF_VARS.fd -a -f OVMF_CODE.fd ]; then
+    UEFI_BIOS=" -drive file=OVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on "
+    UEFI_BIOS+=" -drive file=OVMF_VARS.fd,if=pflash,format=raw,unit=1 "
+fi
+
 VMN=${VMN:=1}
 
 qemu-system-x86_64 \
     -enable-kvm \
-    -bios OVMF.fd \
+    ${UEFI_BIOS} \
     -smp sockets=1,cpus=4,cores=2 -cpu host \
     -m 1024 \
     -vga none -nographic \
     -drive file="$IMAGE",if=virtio,aio=threads,format=raw \
     -netdev user,id=mynet0,hostfwd=tcp::${VMN}0022-:22,hostfwd=tcp::${VMN}2375-:2375 \
     -device virtio-net-pci,netdev=mynet0 \
+    -device virtio-rng-pci \
     -debugcon file:debug.log -global isa-debugcon.iobase=0x402 $@

user-setup.sh

@@ -1,102 +1,136 @@
-#!/bin/sh
+#!/bin/bash
 
 SCRIPT=$(/usr/bin/basename $0)
 PEM=""
 SERVERCA=""
 CLIENTCA=""
+WORKSPACE="clearlinux"
+PACKAGE_REPOS=
+NEEDS_KVM_GROUP=
 
 help() {
-	printf "%s\n" >&2 "Usage: $SCRIPT [options]" \
-			  "" \
-			  "Options:" \
-			  "-k --client-cert PEM_FILE: Enable client user cert for koji configuration; requires a PEM file argument" \
-			  "-s --server-ca PEM_FILE: Enable server CA cert for koji configuration; requires a PEM file argument" \
-			  "-c --client-ca PEM_FILE: Enable client CA cert for koji configuration; requires a PEM file argument" \
-			  ""
+  printf "%s\n" >&2 "Usage: $SCRIPT [options]" \
+    "" \
+    "Options:" \
+    "" \
+    "-d --directory NAME: Set up workspace in the given directory." \
+    "-a --clone-packages: Clone all package repos." \
+    "-j --jobs [NUM]: Clone repos with NUM jobs. If NUM is not given, it is set to the available CPU count." \
+    "" \
+    "-k --client-cert PEM_FILE: Enable client user cert for koji configuration; requires a PEM file argument" \
+    "-s --server-ca PEM_FILE: Enable server CA cert for koji configuration; requires a PEM file argument" \
+    "-c --client-ca PEM_FILE: Enable client CA cert for koji configuration; requires a PEM file argument" \
+    ""
+}
+
+error() {
+  echo -e "Error: $1\n" >&2
+  help
+  exit 1
 }
 
 while [ $# -gt 0 ]; do
-	case "$1" in
-		"--help"|"-h")
-			help
-			exit 0
-			;;
-		"--client-cert"|"-k")
-			shift
-			PEM="$PWD/$1"
-			;;
-		"--server-ca"|"-s")
-			shift
-			SERVERCA="$PWD/$1"
-			;;
-		"--client-ca"|"-c")
-			shift
-			CLIENTCA="$PWD/$1"
-			;;
-		*)
-			help
-			exit 1
-			;;
-	esac
-	shift
+  case "$1" in
+    "--help"|"-h")
+      help
+      exit 0
+      ;;
+    "--client-cert"|"-k")
+      shift
+      PEM="$(realpath $1)"
+      ;;
+    "--server-ca"|"-s")
+      shift
+      SERVERCA="$(realpath $1)"
+      ;;
+    "--client-ca"|"-c")
+      shift
+      CLIENTCA="$(realpath $1)"
+      ;;
+    "--jobs"|"-j")
+      if echo "$2" | grep -qx "[1-9][0-9]*"; then
+        shift
+        JOBS="$1"
+      elif [ -f /proc/cpuinfo ]; then
+        JOBS=$(grep -Ec '^processor.*:.*[0-9]+$' /proc/cpuinfo)
+      fi
+      ;;
+    "--directory"|"-d")
+      [ -z "$2" ] && error "Must supply a directory name to the -d option"
+      [ "${2:0:1}" = "-" ] && error "Directory name cannot begin with \"-\""
+      shift
+      WORKSPACE="$1"
+      ;;
+    "--clone-packages"|"-a")
+      PACKAGE_REPOS=1
+      ;;
+    *)
+      help
+      exit 1
+      ;;
+  esac
+  shift
 done
 
-error() {
-	echo -e "Error: $1\n" >&2
-	help
-	exit 1
-}
 
 if [ -z "$PEM" ] && [ -z "$SERVERCA" ] && [ -z "$CLIENTCA" ]; then
-	USE_KOJI=
+  USE_KOJI=
 else
-	if [ -z "$PEM" ] || [ -z "$SERVERCA" ] || [ -z "$CLIENTCA" ]; then
-		error "Must specify all three command line options (or none)"
-	fi
-	if [ ! -f "$PEM" ]; then
-		error "Missing koji client PEM key file"
-	fi
-	if [ ! -f "$SERVERCA" ]; then
-		error "Missing koji server CA PEM file"
-	fi
-	if [ ! -f "$CLIENTCA" ]; then
-		error "Missing koji client CA PEM file"
-	fi
-	USE_KOJI="yes"
+  if [ -z "$PEM" ] || [ -z "$SERVERCA" ] || [ -z "$CLIENTCA" ]; then
+    error "Must specify all three command line options (or none)"
+  fi
+  if [ ! -f "$PEM" ]; then
+    error "Missing koji client PEM key file"
+  fi
+  if [ ! -f "$SERVERCA" ]; then
+    error "Missing koji server CA PEM file"
+  fi
+  if [ ! -f "$CLIENTCA" ]; then
+    error "Missing koji client CA PEM file"
+  fi
+  USE_KOJI="yes"
+fi
+
+if [ -n "$JOBS" ]; then
+  JOBS_ARG="-j $JOBS"
+fi
+
+if [ -d "$WORKSPACE" ]; then
+  error "Directory \"$WORKSPACE\" already exists. \
+Either remove this workspace, or use a different workspace name."
 fi
 
 required_progs() {
-	local bindir="/usr/bin"
-	for f in git mock rpm rpmbuild ; do
-		[ ! -x "${bindir}/${f}" ] && missing+="${f} "
-	done
-	[ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
-	if [ -n "$missing" ]; then
-		echo "Install the following programs and re-run this script:" >&2
-		echo $missing >&2
-		echo 'All programs should be provided in the "os-clr-on-clr" bundle.' >&2
-		exit 1
-	fi
+  local bindir="/usr/bin"
+  for f in git mock rpm rpmbuild diffstat ; do
+    [ ! -x "${bindir}/${f}" ] && missing+="${f} "
+  done
+  [ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
+  if [ -n "$missing" ]; then
+    echo "Install the following programs and re-run this script:" >&2
+    echo $missing >&2
+    echo 'All programs should be provided in the "os-clr-on-clr" bundle.' >&2
+    exit 1
+  fi
 }
 
 required_progs
 
-echo 'Initializing development workspace in "clearlinux" . . .'
-if [ -d "clearlinux" ]; then
-	echo 'Directory "clearlinux" already exists in current directory.' >&2
-	echo "Cannot initialize workspace." >&2
-	exit 1
+if ! groups | grep -qw kvm; then
+  NEEDS_KVM_GROUP=1
 fi
 
-mkdir clearlinux
-cd clearlinux
+echo "Initializing development workspace in \"$WORKSPACE\" . . ."
+
+mkdir "$WORKSPACE"
+cd "$WORKSPACE"
 
 echo "Setting up common repo . . ."
 mkdir projects
 git clone https://github.com/clearlinux/common projects/common
 if [ $? -ne 0 ]; then
-	echo "Failed to clone common repo." >&2
-	exit 1
+  echo "Failed to clone common repo." >&2
+  exit 1
 fi
 
 # Finish setup for packages/projects hierarchy
@@ -105,48 +139,56 @@ mkdir -p packages/common
 ln -sf ../../projects/common/Makefile.common packages/common/Makefile.common
 
 if [ "$USE_KOJI" ]; then
-	echo "Setting up koji certs . . ."
-	mkdir -p ~/.koji
-	cp "$PEM" ~/.koji/client.crt
-	cp "$CLIENTCA" ~/.koji/clientca.crt
-	cp "$SERVERCA" ~/.koji/serverca.crt
+  echo "Setting up koji certs . . ."
+  mkdir -p ~/.koji
+  cp "$PEM" ~/.koji/client.crt
+  cp "$CLIENTCA" ~/.koji/clientca.crt
+  cp "$SERVERCA" ~/.koji/serverca.crt
 
-	if [ ! -f /etc/koji.conf ]; then
-		echo "Setting up koji config . . ."
-		sudo cp projects/common/koji-client-files/koji.conf /etc
-	fi
+  if [ ! -f ~/.koji/config ]; then
+    echo "Setting up koji config . . ."
+    cp -f projects/common/conf/koji.conf ~/.koji/config
+  fi
 fi
 
-if [ ! -f /etc/mock/clear.cfg ]; then
-	echo "Setting up mock config . . ."
-	sudo mkdir -p /etc/mock
-	sudo cp projects/common/koji-client-files/clear.cfg /etc/mock
+if [ -n "$NEEDS_KVM_GROUP" ]; then
+  echo "Adding user to kvm group . . ."
+  sudo usermod -a -G kvm $USER
 fi
 
-echo "Adding user to kvm group . . ."
-sudo usermod -a -G kvm $USER
-
 echo "Cloning special project repositories . . ."
-make clone-projects
+make ${JOBS_ARG} clone-projects
 
-if [ -z "$NO_PACKAGE_REPOS" ]; then
-	echo "Cloning all package repositories . . ."
-	make clone-packages
+if [ -n "$PACKAGE_REPOS" ]; then
+  echo "Cloning all package repositories . . ."
+  make ${JOBS_ARG} clone-packages
 fi
 
+echo "Creating mix workspace . . ."
+mkdir -p mix
+
 if [ "$USE_KOJI" ]; then
-	echo "Testing koji installation . . ."
-	if koji moshimoshi; then
-		echo -en "\n************************\n\n"
-		echo "Koji installed and configured successfully"
-	else
-		echo -en "\n************************\n\n"
-		echo "Error with koji installation or configuration" >&2
-		exit 1
-	fi
+  echo "Testing koji installation . . ."
+  if koji moshimoshi; then
+    echo -en "\n************************\n\n"
+    echo "Koji installed and configured successfully"
+  else
+    echo -en "\n************************\n\n"
+    echo "Error with koji installation or configuration" >&2
+    exit 1
+  fi
 fi
 
 echo -en "\n************************\n"
 
-echo 'Workspace has been set up in the "clearlinux" directory'
-echo 'NOTE: logout and log back in or run "newgrp kvm" to finalize the setup process'
+echo "Workspace has been set up in \"$WORKSPACE\""
+if [ -z "$PACKAGE_REPOS" ]; then
+  echo "NOTE: To clone all package repos, run \"cd $WORKSPACE; make [-j NUM] clone-packages\""
+  echo "NOTE: To clone a single package repo with NAME, run \"cd $WORKSPACE; make clone_NAME\""
+fi
+if [ -n "$NEEDS_KVM_GROUP" ]; then
+  echo 'NOTE: logout and log back in to finalize the setup process'
+fi
+
+
+# vi: ft=sh sw=2 et sts=2