Update packages file for version 33710

Signed-off-by: clrbuilder <clrbuilder@intel.com>
Update packages file for version 33690
2026-06-16 11:06:15 +00:00 · 2020-09-15 00:09:12 -07:00 · 2020-09-08 11:02:38 -07:00 · 2020-09-03 00:07:17 -07:00 · 2020-09-01 00:07:38 -07:00 · 2020-08-13 00:09:30 -07:00
14 changed files with 1330 additions and 543 deletions
@@ -21,7 +21,8 @@ RPMS ?= $(LATEST_RPMS)
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )

 # MOCK_CONFIG_VAL is set in Makefile.shared
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_OPTS)
+MOCK_SMP_FLAGS=$(patsubst -j%, -D_smp_mflags\ -j%,$(filter -j%, $(MAKEFLAGS)))
+MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_SMP_FLAGS) $(MOCK_OPTS)

 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
@@ -49,13 +50,6 @@ spdxcheck: $(SPECFILE)
 		if ! grep -qx "$$LICENSE" $(TOPLVL)/projects/common/licenses-{spdx,exceptions,extra}; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
 	done; test -z "$$FAIL"

-#help cvecheck: Checks
-#help for common vulnerabilities and exposures in your package.
-cvecheck:
-	# https://github.com/ikeydoherty/cve-check-tool
-	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
-	cve-check-tool -M $(TOPLVL)/projects/common/mapping $(SPECFILE)
-
 #help clean: Cleans up the src.rpm file. but not the rpm files or log files.
 clean:
 	rm -f $(SRPMFILE)
@@ -143,6 +137,10 @@ pullrebase:
 			echo "No commits exist in remote. Continuing."; \
 			exit 0; \
 		fi; \
+		if [ "$$(git rev-list FETCH_HEAD | head -n 1)" = "$$(git rev-list HEAD | head -n 1)" ]; then \
+			echo "Current branch up-to-date. Continuing."; \
+			exit 0; \
+		fi; \
 		echo "Rebasing to origin/master"; \
 		if git status --porcelain | grep -q '^.[^?]'; \
 		then \
@@ -175,9 +173,11 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		--name $(PKG_NAME) \
 		--archives $(ARCHIVES) \
 		--mock-config $(MOCK_CONFIG_VAL) \
-		--mock-opts "$(MOCK_OPTS)" \
+		--mock-opts="$(MOCK_SMP_FLAGS) $(MOCK_OPTS)" \
 		$${SETVERSION:+ --version $${SETVERSION}} \
-		${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+		$${NON_INTERACTIVE:+ --non_interactive} \
+		$${SKIP_GIT:+ --skip-git} \
+		$${CLEANUP:+ -C} \
 		$(firstword $(NEWURL) $(URL));
 	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@$(MAKE) spdxcheck
@@ -187,6 +187,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		git commit --amend --no-edit Change* ; \
 	fi
 	@$(MAKE) -s for-review.txt
+	diffstat for-review.txt
 	@printf "\n**\n"
 	@printf "** NOTICE: A patch with changes is available in the file for-review.txt\n"
 	@printf "** To recreate (e.g., after git commit --amend), run make for-review.txt\n"
@@ -201,12 +202,12 @@ for-review.txt:

 #help autospecnogit: Runs autospec, but does not create a commit
 autospecnogit:
-	$(MAKE) SKIP_GIT=--skip-git autospec
+	$(MAKE) autospec SKIP_GIT=1

 #help autospecnostate: Runs autospec, but cleans up mock chroots
 #help and disables interactive mode.
 autospecnostate:
-	$(MAKE) CLEANUP=-C NON_INTERACTIVE=--non_interactive autospec
+	$(MAKE) autospec CLEANUP=1 NON_INTERACTIVE=1

 scanlicense:
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --config $(AUTOSPEC_CONF) --license-only $(firstword $(NEWURL) $(URL)) --name $(PKG_NAME)
@@ -215,7 +216,7 @@ scanlicense:
 bump:
 	git pull --rebase
 	$(MAKE) bumpnogit
-	git add *.spec release
+	git add $(SPECFILE) release
 	git commit -a -m "version bump from $(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)) to `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)`"

 #help bumpnogit: Increments the release file by one and does not commit changes.
@@ -304,16 +305,21 @@ prekoji-checks:
 koji: prekoji-checks kojidef
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
-	@if ! git branch | grep -q -E '^\* master'; then \
+	@if [ "$$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then \
 		echo "Error: Must be on the master branch to submit to koji" >&2; \
 		exit 1; \
 	fi; \
-	if ! git diff --quiet HEAD ${SPECFILE}; then \
+	if ! git diff --quiet HEAD -- ${SPECFILE}; then \
 		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
 		exit 1; \
 	fi; \
-	if git rev-parse --verify --quiet origin/master > /dev/null; then \
+	git fetch; \
+	if git rev-parse --verify --quiet origin/master >/dev/null; then \
 		git pull --rebase; \
+	fi; \
+	if git ls-remote --tags --exit-code origin $(SRPMVERS) >/dev/null; then \
+		echo "Error: remote tag $(SRPMVERS) already exists" >&2; \
+		exit 1; \
 	fi
 	git tag $(SRPMVERS)
 	git push origin master refs/tags/$(SRPMVERS)
@@ -344,11 +350,6 @@ rekoji-nowait:
 logcheck:
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log

-$(TOPLVL)/repo:
-	@echo "Creating local RPM repository $(TOPLVL)/repo"
-	mkdir $(TOPLVL)/repo
-	$(MAKE) localrepocreate
-
 #help repoadd: Adds locally-built RPMs for this package to the local RPM
 #help repository. If this repository does not yet exist, it is created and
 #help enabled.
@@ -356,39 +357,51 @@ repoadd: $(TOPLVL)/repo
 	@if [ -z "${LATEST_RPMS}" ]; then \
 		echo "No rpms found in rpms/ directory."; \
 		exit 1; \
-	fi; \
-	if [ -f .repo-index ]; then \
-		echo "Cleaning old rpms:"; \
-		for r in `cat .repo-index`; do \
-			echo "-$$r"; \
-			rm -f $(TOPLVL)/repo/$$r; \
-		done; \
-		rm .repo-index; \
-	fi; \
-	for r in ${LATEST_RPMS}; do \
-		echo "+$$r"; \
-		echo $$(basename $$r) >> .repo-index; \
-		ln -f $$r $(TOPLVL)/repo/; \
-	done
-	$(MAKE) localrepocreate
-	$(call localrepoenable,${PM_CONF},${MOCK_CONF})
+	fi
+	$(MAKE) -s repodel NO_CREATEREPO=1
+	@echo "Adding new rpms:"; \
+	for rpm in ${LATEST_RPMS}; do \
+		echo "+$${rpm#rpms/}"; \
+		echo "$${rpm#rpms/}" >> .repo-index; \
+	done; \
+	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
+	$(MAKE) -s localrepocreate
+	$(MAKE) -s localrepoenable
+
+#help repostage: Stages locally-built RPMs for this package to the local RPM
+#help directory. If this directory does not yet exist, it is created and
+#help enabled.
+repostage: $(TOPLVL)/repo
+	@if [ -z "${LATEST_RPMS}" ]; then \
+		echo "No rpms found in rpms/ directory."; \
+		exit 1; \
+	fi
+	$(MAKE) -s repodel NO_CREATEREPO=1
+	@for r in ${LATEST_RPMS}; do \
+		echo "+$${rpm#rpms/}"; \
+		echo "$${rpm#rpms/}" >> .repo-index; \
+	done; \
+	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}

 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
-repodel:
+repodel: $(TOPLVL)/repo
 	@if [ -f .repo-index ]; then \
 		echo "Cleaning old rpms:"; \
-		for r in `cat .repo-index`; do \
-			echo "-$$r"; \
-			rm -f $(TOPLVL)/repo/$$r; \
-		done; \
+		mapfile -t rpms < <(sed 's|.*|$</&|' .repo-index); \
+		for rpm in "$${rpms[@]}"; do \
+			echo "-$${rpm#$</}"; \
+		done < .repo-index; \
+		flock $</repo.lock rm -f "$${rpms[@]}"; \
 		rm .repo-index; \
-		$(MAKE) localrepocreate; \
+		if [ -z "$$NO_CREATEREPO" ]; then \
+			$(MAKE) -s localrepocreate; \
+		fi; \
 	fi

 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
-repoenable: $(TOPLVL)/repo ; $(call localrepoenable,${PM_CONF},${MOCK_CONF})
+repoenable: localrepoenable ;

 #help repodisable: Disables the local RPM repository.
 repodisable: localrepodisable ;
@@ -62,33 +62,95 @@ define remotepkgname
 $(if $(USE_PACKAGE_MAPPING),$$(awk -v P="$(1)" '$$1 == P { res=$$2 } END { print res ? res : P }' $(TOPLVL)/projects/common/pkg-mapping),$(1))
 endef

+# Clone repo from the configured gitolite instance, but only if the repo exists
+# on that instance and matches the pattern "packages/..*".
+define try-clone-gitolite
+rpkg=$(call remotepkgname,$(1)); \
+if ssh $(GITOLITE_BASE_URL) info "packages/$$rpkg" | grep --quiet "packages/$$rpkg$$"; then \
+	git clone $(PKG_BASE_URL)/$$rpkg packages/$(1); \
+	cd packages/$(1); \
+	$(call gitoliteurl,packages/$(1)); \
+	$(call subjectprefix,$(1)); \
+fi
+endef
+
+# Clone repo from the configured package repo hosting location, as defined by
+# the PKG_BASE_URL variable, but only if the repo exists.
+define try-clone-other
+rpkg=$(call remotepkgname,$(1)); \
+if git ls-remote $(PKG_BASE_URL)/$$rpkg &> /dev/null; then \
+	git clone $(PKG_BASE_URL)/$$rpkg packages/$(1); \
+	cd packages/$(1); \
+	$(call subjectprefix,$(1)); \
+fi
+endef
+
+# First, try running `make clone_PKG`, where PKG is the first argument. If that
+# fails, proceed with fallback cloning strategies.
+define clone-if-available
+$(MAKE) clone_$(1) || { \
+	$(if $(GITOLITE_BASE_URL),$(call try-clone-gitolite,$(1)),$(call try-clone-other,$(1))); \
+}
+endef
+
+$(TOPLVL)/repo:
+	@echo "Creating local RPM repository $(TOPLVL)/repo"
+	mkdir $(TOPLVL)/repo
+
 # Enables the local RPM repo by installing package manager and Mock configs.
 # First argument is the path to the package manager config, and the second
 # argument is the path to the mock config. Each of these config files is
 # augmented for local repo support.
-define localrepoenable
-	@cp $(1) $(TOPLVL)/repo/yum.conf
-	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/yum.conf
-	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/yum.conf
-	@cp $(2) $(TOPLVL)/repo/clear.cfg
-	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/clear.cfg
-	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/clear.cfg
-	@sed -i '/^"""/d; $$a"""' $(TOPLVL)/repo/clear.cfg
-endef
+localrepoenable: $(TOPLVL)/repo
+	@if grep -qx '\[localrepo\]' ${PM_CONF}; then \
+		echo "[ERROR] ${PM_CONF} contains section named [localrepo]."; \
+		echo "[ERROR] Remove that section and try again."; \
+		exit 1; \
+	fi >&2
+	@if grep -qx '\[localrepo\]' ${MOCK_CONF}; then \
+		echo "[ERROR] ${MOCK_CONF} contains section named [localrepo]."; \
+		echo "[ERROR] Remove that section and try again."; \
+		exit 1; \
+	fi >&2
+	@last="$$(tail -n 1 ${MOCK_CONF})"; \
+	if [[ "$$last" != "\"\"\"" ]]; then \
+		echo "[ERROR] Last line of ${MOCK_CONF} is malformed; expected: \"\"\"; actual: $$last"; \
+		echo "[ERROR] Fix that line and try again."; \
+		exit 1; \
+	fi >&2
+	@pm_new=$$(mktemp -p $< yum.conf.XXXXXXX); \
+	pm_orig=$</yum.conf; \
+	mock_new=$$(mktemp -p $< clear.cfg.XXXXXXX); \
+	mock_orig=$</clear.cfg; \
+	cp ${PM_CONF} $$pm_new; \
+	printf "\n\n[localrepo]\nname=localrepo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $$pm_new; \
+	printf "baseurl=file://$$(realpath $<)/\n\n" >> $$pm_new; \
+	if ! cmp -s $$pm_new $$pm_orig; then \
+		flock $</repo.lock mv $$pm_new $$pm_orig; \
+	fi; \
+	rm -f $$pm_new; \
+	cp ${MOCK_CONF} $$mock_new; \
+	sed -i '$$d' $$mock_new; \
+	printf "\n\n[localrepo]\nname=localrepo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $$mock_new; \
+	printf "baseurl=file://$$(realpath $<)/\n\n" >> $$mock_new; \
+	sed -i '$$a"""' $$mock_new; \
+	if ! cmp -s $$mock_new $$mock_orig; then \
+		flock $</repo.lock mv $$mock_new $$mock_orig; \
+	fi; \
+	rm -f $$mock_new

 # Disables the local RPM repo by removing the installed package manager and
 # Mock configuration files.
-localrepodisable:
-	@rm -f $(TOPLVL)/repo/yum.conf
-	@rm -f $(TOPLVL)/repo/clear.cfg
+localrepodisable: $(TOPLVL)/repo
+	flock $</repo.lock rm -f $</yum.conf $</clear.cfg

-# Runs createrepo_c (or createrepo) for the local RPM repo
-localrepocreate:
-	( cd $(TOPLVL)/repo ; if type createrepo_c 1>/dev/null 2>&1; then createrepo_c .; else createrepo .; fi );
+# Runs createrepo_c for the local RPM repo
+localrepocreate: $(TOPLVL)/repo
+	cd $< && flock repo.lock createrepo_c .

 # Remove all RPMs from the local RPM repo
-localrepoclean:
-	@find $(TOPLVL)/repo -maxdepth 1 -name '*.rpm' -delete
+localrepoclean: $(TOPLVL)/repo
+	flock $</repo.lock find $< -maxdepth 1 -name '*.rpm' -delete

 # Summarizes the local RPM repo status
 localrepostatus:
@@ -34,7 +34,7 @@ $(proj_PKGS):
 	git clone $(PRJ_BASE_URL)/$$proj projects/$$proj; \
 	cd projects/$$proj; \
 	if [ "$$proj" = "autospec" ] ; then \
-		git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+		git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 	else \
 		$(call gitoliteurl,projects/$$proj); \
 	fi; \
@@ -56,12 +56,6 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	$(call subjectprefix,$$pkg); \
 	)

-#help cvecheck: Checks for common vulnerabilities and exposures in your code.
-cvecheck:
-	# https://github.com/ikeydoherty/cve-check-tool
-	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
-	cve-check-tool -n -M $(TOPLVL)/projects/common/mapping $(PACKAGES_FILE)
-
 #help pull: Performs a git pull --rebase for each package repo, avoiding the creation
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
@@ -83,7 +77,7 @@ ${PULL_PKGS}:
 			git --no-pager log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit $$O..$$N; echo; \
 		fi ; \
 		if [ "$$p" = "projects/autospec" ] ; then \
-			git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+			git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 		else \
 			$(call gitoliteurl,$$p); \
 		fi ; \
@@ -172,6 +166,15 @@ diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 		(cd $$p ; git status | grep -q 'nothing to commit, working [^ ]* clean' || (echo "Uncommitted changes in: $$p"; git diff |diffstat -p1)) ;\
 	done

+#help log: Run git log origin/master..HEAD for all package repos, thus
+#help displaying any committed but unpushed changes.
+log: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
+	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
+		( cd $$p; if [ "$$(git rev-parse --verify HEAD)" != "$$(git rev-parse --verify origin/master)" ] ; then \
+		echo $$p ; git --no-pager log --pretty="format:%s" origin/master..HEAD ; echo ; echo;\
+		fi ; ) \
+	done
+
 #help versions: Displays the version of each package in the ./packages tree.
 versions: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
@@ -202,7 +205,7 @@ autospecnew: preautospecnew-checks localreponotice
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
-	-$(MAKE) clone_$(NAME)
+	$(call clone-if-available,$(NAME))
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
@@ -221,7 +224,7 @@ autospecnew: preautospecnew-checks localreponotice
 			--name $(NAME) \
 			--archives $(ARCHIVES) \
 			--mock-config $(MOCK_CONFIG_VAL) \
-			--mock-opts "$(MOCK_OPTS)" \
+			--mock-opts="$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
 			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 			$(URL); \
@@ -233,20 +236,63 @@ autospecnew: preautospecnew-checks localreponotice
 			printf "** NOTICE: A patch with changes is available in the file $(TOPLVL)/packages/$(NAME)/for-review.txt\n"; \
 			printf "** Please look through this file and if you are unsure, please submit for code review with git send-email\n"; \
 			printf "**\n\n"; \
+		else \
+			echo "Autospec of $(NAME) failed."; \
+			exit 1; \
 		fi; \
 	else \
 		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
 		exit 1; \
 	fi

-$(TOPLVL)/repo:
-	@echo "Creating local RPM repository $(TOPLVL)/repo"
-	mkdir $(TOPLVL)/repo
-	$(MAKE) localrepocreate
+#help autospecnewgo: Creates a new autospec package with (dependencies)
+#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
+#help autospec will be created in the help process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
+autospecnewgo: preautospecnew-checks localreponotice
+	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
+		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
+		exit 1; \
+	fi
+	-$(MAKE) clone_$(NAME)
+	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
+		echo "no remote repository found, creating new package repository and running autospec"; \
+		mkdir -p $(TOPLVL)/packages/$(NAME); \
+		( \
+			cd $(TOPLVL)/packages/$(NAME); \
+			git init; \
+			git remote add origin $(PKG_BASE_URL)/$(NAME); \
+			$(call gitoliteurl,packages/$(NAME)); \
+			$(call subjectprefix,$(NAME)); \
+		); \
+		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
+		python3 $(TOPLVL)/projects/common/go-builder.py \
+			--target packages/$(NAME) \
+			--integrity \
+			--config "$(AUTOSPEC_CONF)" \
+			--name $(NAME) \
+			--archives $(ARCHIVES) \
+			--mock-config $(MOCK_CONFIG_VAL) \
+			--mock-opts "$(MOCK_OPTS)" \
+			$(URL); \
+		if [ $$? -eq 0 ]; then \
+			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
+			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
+			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
+			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
+		else \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
+		exit 1; \
+	fi

 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
-repoenable: $(TOPLVL)/repo ; $(call localrepoenable,${PM_CONF},${MOCK_CONF})
+repoenable: localrepoenable ;

 #help repodisable: Disables the local RPM repository.
 repodisable: localrepodisable ;
@@ -0,0 +1,40 @@
+## Advanced topics
+
+### Manual setup
+
+If you did not run the user-setup script (see "Automated setup" section in the
+[main README](README.md), you will want to set up the developer tooling
+workspace manually. This section provides general documentation for the manual
+setup process, and it is not meant to be exhaustive.
+
+On your Clear Linux system, create a workspace for Clear Linux development
+work:
+
+```
+$ mkdir clearlinux
+```
+
+Clone this repo into a `projects` directory within the workspace:
+
+```
+$ cd clearlinux
+$ mkdir projects
+$ git clone https://github.com/clearlinux/common projects/common
+```
+
+Create the toplevel tooling Makefile:
+
+```
+$ ln -s projects/common/Makefile.toplevel Makefile
+```
+
+Clone all Clear Linux package and project repositories:
+
+```
+$ make clone
+```
+
+Note: You can clone the repos in parallel by using make's `-j` option.
+
+At this point, the `packages` directory will contain all Clear Linux package
+repos, and `projects` will contain common, clr-bundles, and autospec repos.
@@ -152,39 +152,4 @@ If `Makefile.config.site_local` doesn't exist already, create it.

 ### Manual setup

-If you did not run the user-setup script (see "Automated setup" section above),
-you will want to set up the developer tooling workspace manually. This section
-provides general documentation for the manual setup process, and it is not
-meant to be exhaustive.
-
-On your Clear Linux system, create a workspace for Clear Linux development
-work:
-
-```
-$ mkdir clearlinux
-```
-
-Clone this repo into a `projects` directory within the workspace:
-
-```
-$ cd clearlinux
-$ mkdir projects
-$ git clone https://github.com/clearlinux/common projects/common
-```
-
-Create the toplevel tooling Makefile:
-
-```
-$ ln -s projects/common/Makefile.toplevel Makefile
-```
-
-Clone all Clear Linux package and project repositories:
-
-```
-$ make clone
-```
-
-Note: You can clone the repos in parallel by using make's `-j` option.
-
-At this point, the `packages` directory will contain all Clear Linux package
-repos, and `projects` will contain common, clr-bundles, and autospec repos.
+See the [Manual setup](README-advanced.md#manual-setup) documentation.
@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+#
+# gowrap.py - part of autospec
+# Copyright (C) 2019 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def parse_args():
+    """Parse commandline arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url", help="Location of package archive")
+    parser.add_argument("-t", "--target", dest="target", action="store",
+                        default=None,
+                        help="Target location to create or reuse")
+    parser.add_argument("-c", "--config", dest="config", action="store",
+                        default="/usr/share/defaults/autospec/autospec.conf",
+                        help="Set configuration file to use")
+    parser.add_argument("-n", "--name", action="store", dest="name", default="",
+                        help="Override the package name")
+    parser.add_argument("-i", "--integrity", action="store_true",
+                        default=False,
+                        help="Search for package signature from source URL and "
+                             "attempt to verify package")
+    parser.add_argument("-m", "--mock-config", action="store", default="clear",
+                        help="Value to pass with Mock's -r option. Defaults to "
+                             "\"clear\", meaning that Mock will use "
+                             "/etc/mock/clear.cfg.")
+    parser.add_argument("-o", "--mock-opts", action="store", default="",
+                        help="Arbitrary options to pass down to mock when "
+                        "building a package.")
+    parser.add_argument('-a', "--archives", action="store",
+                        dest="archives", default=[], nargs='*',
+                        help="tarball URLs for additional source archives and"
+                        " a location for the sources to be extacted to (e.g."
+                        " http://example.com/downloads/dependency.tar.gz"
+                        " /directory/relative/to/extract/root )")
+
+    return parser.parse_args()
+
+
+def missing_dependencies(name):
+    """Check if the build failed due to missing dependencies."""
+    root_log = os.path.join("packages", name, "results", "root.log")
+    if not os.path.exists(root_log):
+        return False
+    with open(root_log, "r") as lfile:
+        for line in lfile.readlines():
+            if "No matching package to install" in line:
+                return True
+    return False
+
+
+def already_built(name, version):
+    """Check if the package and version are already built."""
+    if not os.path.exists(os.path.join("packages", name, "rpms")):
+        return False
+    vpath = os.path.join("packages", name, "versions")
+    if not os.path.exists(vpath):
+        return False
+    with open(vpath, "r") as vfile:
+            if version not in [x.strip() for x in vfile.readlines()]:
+                return False
+    return True
+
+
+def build(path, args, name, url, version):
+    """Try and build a package."""
+    if version:
+        ver_arg = ["-v", version]
+    else:
+        ver_arg = []
+    if already_built(name, version):
+        return True
+    print(f"Trying to autospec {name} - {version}")
+
+    proc = subprocess.run(["python3",
+                           f"{path}/../autospec/autospec/autospec.py", url,
+                           "-c", args.config, "-t", f"packages/{name}",
+                           "-n", name, "-m", args.mock_config,
+                           "-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
+    if proc.returncode == 0:
+        subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
+                       capture_output=True)
+    return proc.returncode == 0
+
+
+def parse_go_mod(path):
+    """Parse go.mod file for build requirements.
+
+    File content looks as follows:
+
+    module example.com/foo/bar
+
+    require (
+        github.com/BurntSushi/toml v0.3.1
+        git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
+        github.com/inconshreveable/mousetrap v1.0.0 // indirect
+        "github.com/spf13/cobra" v0.0.3
+        github.com/spf13/pflag v1.0.3 // indirect
+    )
+
+    Need to handle all require lines including //indirect.
+    Skip requires that use .git for now. May need to be handled
+    differently.
+    """
+    reqs = []
+    with open(path, "r") as gfile:
+        dep_start = False
+        for line in gfile.readlines():
+            # Ideally the mod file is generated and the format is
+            # always correct but add a few defenses just in case
+            line = line.strip()
+            if line.startswith("//"):
+                # Skip comments
+                continue
+            if dep_start:
+                # End of the require section
+                if line.startswith(")"):
+                    break
+                req = line.split()[:2]
+                req[0] = req[0].replace('"', '')
+                if req[0].endswith(".git"):
+                    continue
+                reqs.append(req)
+                continue
+            if line.startswith("require ("):
+                dep_start = True
+    return reqs
+
+
+def get_dependencies(name):
+    """Return path to the go.mod file if it exists."""
+    command = None
+    path = None
+    pdir = os.path.join("packages", name)
+    reqs = []
+    for fname in os.listdir(pdir):
+        # Find the archive
+        if fname.endswith("zip"):
+            command = ["unzip", fname]
+        elif ".tar." in fname:
+            command = ["tar", "xf", fname]
+        else:
+            command = None
+
+        if command:
+            # Decompress archive
+            tdir = tempfile.mkdtemp()
+            tfile = os.path.join(tdir, fname)
+            shutil.copyfile(os.path.join(pdir, fname), tfile)
+            proc = subprocess.run(command, cwd=tdir, capture_output=True)
+            if not proc.returncode:
+                # inspect contents for a "go.sum" file
+                for root, _, files in os.walk(tdir):
+                    if "go.mod" in files:
+                        path = f"{name}.gomod"
+                        shutil.copyfile(os.path.join(root, "go.mod"), path)
+                        reqs += parse_go_mod(path)
+                        os.unlink(path)
+            shutil.rmtree(tdir)
+
+    return reqs
+
+
+def encode_requirements(requirement):
+    """Encode the project's module and version."""
+    # Default dependency base proxy url
+    base_url = "https://proxy.golang.org/"
+    # Encoding based on
+    # https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
+    encoded_module = ""
+    for char in requirement[0]:
+        if char.isupper():
+            encoded_module += "!" + char.lower()
+        else:
+            encoded_module += char
+    encoded_version = ""
+    for char in requirement[1]:
+        if char.isupper():
+            encoded_version += "!" + char.lower()
+        else:
+            encoded_version += char
+    return (encoded_module, encoded_version)
+
+
+def initialize_package(name, url, version, path, args):
+    """Setup package and try to build it."""
+    if already_built(name, version):
+        return True
+    print(f"Trying to autospecnew {name} - {version}")
+
+    proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
+    if f"{name} already exists at" in proc.stdout.decode("utf-8"):
+        return build(path, args, name, url, version)
+    return proc.returncode == 0
+
+
+def build_recursive(path, args, name, url, version=None, success=None):
+    """Try and recursively build packages."""
+    if not success:
+        # First package being built, common utilities already run
+        # so just use autospec directly
+        success = {}
+        ret = build(path, args, name, url, version)
+    else:
+        ret = initialize_package(name, url, version, path, args)
+
+    if ret:
+        # Completed so we are done!
+        return True
+
+    if not missing_dependencies(name):
+        # Failed but not due to missing dependencies, bail
+        print(f"Unknown error building {name} - {version}")
+        return False
+
+    reqs = get_dependencies(name)
+    if not reqs:
+        # Didn't try and add anything so don't need to build again
+        print(f"Build failed with no missing requirements: {name} - {version}")
+        return False
+
+    print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
+    for req in reqs:
+        ereq = encode_requirements(req)
+        rname = "go-" + req[0].replace("/", "-")
+        rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
+        if (rname, req[1]) in success:
+            # Rebuild detected, figure out if it is okay or not
+            if success[(rname, req[1])]:
+                # Already succeeded building req short circuit success
+                continue
+            else:
+                # Encountered a build loop, bail out as this likely
+                # requires manual fixing
+                print(f"Detected build loop when building {rname} - {req[1]}")
+                return True
+        success[(rname, req[1])] = False
+        if not build_recursive(path, args, rname, rurl, req[1], success):
+            print(f"Failed to build dependency {rname} - {req[1]}")
+            return False
+
+        success[(rname, req[1])] = True
+        subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
+        subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
+
+    # Create the repo with the new packages added
+    subprocess.run(["make", "localrepocreate"], capture_output=True)
+    # Retry previously failed build assuming dependencies got added
+    print(f"Rebuilding after resolving dependencies {name} - {version}")
+    ret = build(path, args, name, url, version)
+    print("")
+
+    return ret
+
+
+def main():
+    """Start program execution."""
+    args = parse_args()
+    path = os.path.dirname(os.path.realpath(__file__))
+    return build_recursive(path, args, args.name, args.url)
+
+if __name__ == '__main__':
+    if not main():
+        sys.exit(-1)
+    print("\n\nBuild completed\n\n")
@@ -1,4 +1,5 @@
 Copyright
 Distributable
+ISSL
 Public-Domain
 sendmail
@@ -46,6 +46,7 @@ BSD-3-Clause-LBNL
 BSD-3-Clause-No-Nuclear-License
 BSD-3-Clause-No-Nuclear-License-2014
 BSD-3-Clause-No-Nuclear-Warranty
+BSD-3-Clause-Open-MPI
 BSD-4-Clause
 BSD-4-Clause-UC
 BSD-Protection
@@ -56,6 +57,7 @@ Barr
 Beerware
 BitTorrent-1.0
 BitTorrent-1.1
+BlueOak-1.0.0
 Borceux
 CATOSL-1.1
 CC-BY-1.0
@@ -88,6 +90,7 @@ CC-BY-SA-2.0
 CC-BY-SA-2.5
 CC-BY-SA-3.0
 CC-BY-SA-4.0
+CC-PDDC
 CC0-1.0
 CDDL-1.0
 CDDL-1.1
@@ -234,6 +237,7 @@ MTLL
 MakeIndex
 MirOS
 Motosoto
+MulanPSL-1.0
 Multics
 Mup
 NASA-1.3
@@ -248,6 +252,7 @@ NPL-1.1
 NPOSL-3.0
 NRL
 NTP
+NTP-0
 Naumen
 Net-SNMP
 NetCDF
@@ -260,7 +265,12 @@ OCLC-2.0
 ODC-By-1.0
 ODbL-1.0
 OFL-1.0
+OFL-1.0-RFN
+OFL-1.0-no-RFN
 OFL-1.1
+OFL-1.1-RFN
+OFL-1.1-no-RFN
+OGL-Canada-2.0
 OGL-UK-1.0
 OGL-UK-2.0
 OGL-UK-3.0
@@ -293,6 +303,8 @@ OpenSSL
 PDDL-1.0
 PHP-3.0
 PHP-3.01
+PSF-2.0
+Parity-6.0.0
 Plexus
 PostgreSQL
 Python-2.0
@@ -311,12 +323,17 @@ SCEA
 SGI-B-1.0
 SGI-B-1.1
 SGI-B-2.0
+SHL-0.5
+SHL-0.51
 SISSL
 SISSL-1.2
 SMLNJ
 SMPPL
 SNIA
 SPL-1.0
+SSH-OpenSSH
+SSH-short
+SSPL-1.0
 SWL
 Saxpath
 Sendmail
@@ -336,6 +353,7 @@ TORQUE-1.1
 TOSL
 TU-Berlin-1.0
 TU-Berlin-2.0
+UCL-1.0
 UPL-1.0
 Unicode-DFS-2015
 Unicode-DFS-2016
@@ -365,6 +383,7 @@ Zend-2.0
 Zimbra-1.3
 Zimbra-1.4
 Zlib
+blessing
 bzip2-1.0.5
 bzip2-1.0.6
 copyleft-next-0.3.0
@@ -374,10 +393,12 @@ diffmark
 dvipdfm
 eCos-2.0
 eGenix
+etalab-2.0
 gSOAP-1.3b
 gnuplot
 iMatix
 libpng-2.0
+libselinux-1.0
 libtiff
 mpich2
 psfrag
@@ -51,4 +51,5 @@ rack = rubygem-rack
 compute = nova
 international_components_for_unicode = icu4c
 node.js = nodejs
-jupyter-core = jupyter_core
+jupyter-core = jupyter_core
+flatten-json = flatten_json
@@ -102,6 +102,22 @@ def zap_entire_file_end(filename):
        if file.endswith(filename):
            files.remove(file)                
            
+def zap_line_in_file_substring(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_delete = list()
+                for line in chunk:
+                    if match in line:
+                        to_delete.append(line)
+                for line in to_delete:
+                    chunk.remove(line)
+            
+
 def zap_line_in_file(filename, match):
    global header
    global files
@@ -209,6 +225,11 @@ def main():
    zap_entire_file_end("xz.sig")
    zap_entire_file_end("gz.sig")
    zap_entire_file_end("bz2.sig")
+    zap_entire_file_end("xz.asc")
+    zap_entire_file_end("gz.asc")
+    zap_entire_file_end("bz2.asc")
+    
+    zap_entire_file_end(".pkey")
    
    zap_line_in_file("b/testresults", "Total : 0")
    zap_line_in_file("b/testresults", "Pass : 0")
@@ -223,15 +244,15 @@ def main():
    
    zap_line_in_file_start(".spec", "Release  :")
    zap_line_in_file_start(".spec", "Source99 :")
-    zap_line_in_file_start(".spec", "Source0  :")
    zap_line_in_file_start(".spec", "export SOURCE_DATE_EPOCH")
-    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
    zap_line_in_file_start(".spec", "export AR=gcc-ar")
    zap_line_in_file_start(".spec", "export RANLIB=gcc-ranlib")
    zap_line_in_file_start(".spec", "export NM=gcc-nm")
    zap_line_in_file_start(".spec", "export CFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export FCFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FCFLAGS=\"$FFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export FFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FFLAGS=\"$FFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "export CXXFLAGS=\"$CXXFLAGS -O3 -ffat-lto-objects -flto=4 \"")
    zap_line_in_file_start(".spec", "%setup -q -n ")
    zap_line_in_file_start(".spec", "URL      :")
@@ -265,12 +286,18 @@ def main():
    zap_line_in_file_start(".spec", "%description dev")
    zap_line_in_file_start(".spec", "%description license")
    zap_line_in_file_start(".spec", "%package doc")
+    zap_line_in_file_start(".spec", "cp -a ")
+    zap_line_in_file_start(".spec", "cd %{_builddir}/")
+    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
+    zap_line_in_file_start(".spec", "rm -rf %{buildroot}")
 
    zap_line_in_file_start("b/Makefile", "URL =")
    zap_line_in_file_start("b/Makefile", "PKG_NAME :=")

+    zap_line_in_file_start("b/options.conf", "name = ")
    zap_line_in_file("b/options.conf", "archives = ")
    zap_line_in_file("b/options.conf", "giturl = ")
+    zap_line_in_file_start("b/options.conf", "url = ")
    zap_line_in_file("b/options.conf", "[package]")
    zap_line_in_file("b/options.conf", "")
    zap_line_in_file("b/options.conf", "[autospec]")
@@ -287,11 +314,13 @@ def main():
    zap_line_in_file("b/options.conf", "# disable parallelization during build")
    zap_line_in_file("b/options.conf", "broken_parallel_build = false")
    zap_line_in_file("b/options.conf", "# this package is a library compatability package and only ships versioned library files")
+    zap_line_in_file("b/options.conf", "# this package is a library compatibility package and only ships versioned library files")
    zap_line_in_file("b/options.conf", "compat = false")
    zap_line_in_file("b/options.conf", "# set conservative build flags")
    zap_line_in_file("b/options.conf", "conservative_flags = false")
    zap_line_in_file("b/options.conf", "# dev package requires the extras to be installed")
    zap_line_in_file("b/options.conf", "dev_requires_extras = false")
+    zap_line_in_file("b/options.conf", "nodebug = false")
    zap_line_in_file("b/options.conf", "# pass -ffast-math to compiler")
    zap_line_in_file("b/options.conf", "fast-math = false")
    zap_line_in_file("b/options.conf", "# optimize build for speed over size")
@@ -321,9 +350,17 @@ def main():
    zap_line_in_file("b/options.conf", "# add clang flags")
    zap_line_in_file("b/options.conf", "use_clang = false")
    zap_line_in_file("b/options.conf", "# configure build for lto")
-    zap_line_in_file("b/options.conf", "use_lto = false")
+    zap_line_in_file("b/options.conf", "use_lto = true")
    zap_line_in_file("b/options.conf", "# require package verification for build")
    zap_line_in_file("b/options.conf", "verify_required = true")
+    zap_line_in_file("b/options.conf", "# do not generate debuginfo for this package")
+    zap_line_in_file("b/options.conf", "# configure build also for openmpi")
+    zap_line_in_file("b/options.conf", "openmpi = false")
+    zap_line_in_file("b/options.conf", "alias =")
+    zap_line_in_file("b/options.conf", "alias = ")
+    zap_line_in_file("b/options.conf", "nodebug = false")
+    zap_line_in_file("b/options.conf", "domain =")
+    zap_line_in_file("b/options.conf", "domain = ")

    zap_line_in_file("b/buildreq_add", "# This file contains additional build requirements that did not get")
    zap_line_in_file("b/buildreq_add", "# picked up automatically. One name per line, no whitespace.")
@@ -349,7 +386,11 @@ def main():
    zap_plus_line_in_file("symbols32")

    zap_entire_file("b/whatrequires")
+    zap_entire_file("b/versions")
+    zap_entire_file("b/.gitignore")
+    

+    zap_line_in_file_substring(".spec", "%{buildroot}/usr/share/package-licenses")   # version number change in license copy
    zap_empty_chunks()
    
    print_all()
@@ -102,7 +102,7 @@ fi

 required_progs() {
  local bindir="/usr/bin"
-  for f in git mock rpm rpmbuild ; do
+  for f in git mock rpm rpmbuild diffstat ; do
    [ ! -x "${bindir}/${f}" ] && missing+="${f} "
  done
  [ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
@@ -145,9 +145,9 @@ if [ "$USE_KOJI" ]; then
  cp "$CLIENTCA" ~/.koji/clientca.crt
  cp "$SERVERCA" ~/.koji/serverca.crt

-  if [ ! -f /etc/koji.conf ]; then
+  if [ ! -f ~/.koji/config ]; then
    echo "Setting up koji config . . ."
-    sudo cp projects/common/conf/koji.conf /etc
+    cp -f projects/common/conf/koji.conf ~/.koji/config
  fi
 fi