Update packages file for version 22190

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile.common

@@ -4,8 +4,8 @@ all: build
 
 TOPLVL = ../..
 
-include $(TOPLVL)/projects/common/Makefile.functions
 include $(TOPLVL)/projects/common/Makefile.config
+include $(TOPLVL)/projects/common/Makefile.shared
 
 SHELL  = /bin/bash
 
@@ -19,12 +19,9 @@ RPMS ?= $(LATEST_RPMS)
 
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
 
-ifeq ($(wildcard $(TOPLVL)/repo/clear.cfg),)
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONF)
-else
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(TOPLVL)/repo/clear.cfg
-USE_LOCAL_REPO = 1
-endif
+# MOCK_CONFIG_VAL is set in Makefile.shared
+MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL)
+
 
 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
@@ -118,28 +115,29 @@ shell:
 
 # Always rebuild the source rpm
 .PHONY: $(SRPMFILE)
-$(SRPMFILE): $(SPECFILE) $(__allsources)
+$(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice
 	@$(MAKE) spdxcheck
-	@if [ -n "${USE_LOCAL_REPO}" ]; then \
-		printf '\n\n** NOTICE: Using local repo for the build. Continuing after 3 seconds.\n'; \
-		printf '** If this is not desired, run `make repodisable`.\n'; \
-		printf '** The repo can be re-enabled later with `make repoenable`.\n'; \
-		printf '** Run `touch $(TOPLVL)/repo/opt-in-build` to prevent the 3 second delay.\n\n\n'; \
-		[ -f $(TOPLVL)/repo/opt-in-build ] || sleep 3; \
-	fi >&2
 	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
 
-# Do a git fetch and a git rebase rather than a git pull so aborting
-# the git pull at a password prompt doesn't leave local changes stashed
-# Do the the git status dance so we only apply a stash we create
+# Do a git fetch and a git rebase to apply local commits on top of latest
+# commits from the remote. A git fetch/rebase is used rather than a git pull so
+# aborting the git pull at a password prompt doesn't leave local changes
+# stashed. Do the git status dance so we only apply a stash we create. Note
+# that a fetch/rebase is not attempted if the remote 'origin' is not
+# initialized locally, or if the remote contains no commits.
 pullrebase:
-	if [ -z "$$SKIP_GIT" ] && [ -e .git ]; then \
-		set -x ; \
+	@if [ -z "$$SKIP_GIT" ] && [ -e .git ]; then \
+		if ! git rev-parse --verify --quiet origin/master > /dev/null; then \
+			echo "Remote not yet initialized. Continuing."; \
+			exit 0; \
+		fi; \
+		echo "Fetching origin"; \
 		git fetch; \
 		if ! git rev-parse --verify --quiet FETCH_HEAD > /dev/null; then \
 			echo "No commits exist in remote. Continuing."; \
 			exit 0; \
 		fi; \
+		echo "Rebasing to origin/master"; \
 		if git status --porcelain | grep -q '^.[^?]'; \
 		then \
 			git stash save; \
@@ -154,17 +152,23 @@ pullrebase:
 #help already a specfile, it will be overwritten. Several files used by
 #help autospec will be created in the process. For more information, see
 #help the project at https://github.com/clearlinux/autospec
-autospec: pullrebase
+autospec: pullrebase localreponotice
 	rm -f results/*src.rpm
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
-	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --integrity \
-		${NON_INTERACTIVE} --config "$(AUTOSPEC_CONF)" ${SKIP_GIT} \
-		$(firstword $(NEWURL) $(URL)) --name $(PKG_NAME) ${CLEANUP} \
-		$${SETVERSION:+ -v $${SETVERSION}} -a $(ARCHIVES)
+	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
+		--target . \
+		--integrity \
+		--config "$(AUTOSPEC_CONF)" \
+		--name $(PKG_NAME) \
+		--archives $(ARCHIVES) \
+		--mock-config $(MOCK_CONFIG_VAL) \
+		$${SETVERSION:+ --version $${SETVERSION}} \
+		${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+		$(firstword $(NEWURL) $(URL));
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
 	@if [ -e update_changelog.sh ] && [ -z "$$SKIP_GIT" ] ; then \
@@ -201,30 +205,12 @@ bumpnogit:
 
 $(__missingsources): sources
 
-#help sources: If SOURCES_URL is defined, download required upstream source
-#help files from that location. Otherwise, try to extract source files from the
-#help SRPM in the latest Clear Linux release, as specified by the LATEST_SRPMS
-#help variable. This will run automatically, as a dependency. NOTE: A Koji
-#help server can make use of this "make sources" command, since it lives in a
-#help repo named "common". If you use this makefile in Koji, ensure
-#help "make sources" continues to work for both remote and local builds, since
-#help it is a prerequisite of several commands for building packages.
-sources: upstream
-ifneq ($(strip $(SOURCES_URL)),)
-	while read u; do \
-		case "$$u" in \
-		    "")    continue ;; \
-		    *://*) n="$$u" ;; \
-		    *)     n="$(SOURCES_URL)/$$u" ;; \
-		esac; \
-		curl --fail --noproxy "*" "$$n" -o `basename "$$n"`; \
-	done < upstream
-else
-	@tmp=$$(mktemp -d -p "."); \
+# First argument is a Clear Linux build number (e.g. 22000)
+define fetch-from-srpm
+	tmp=$$(mktemp -d -p "."); \
 	srpm=$$(rpmspec --srpm -q --qf "%{NVR}.src.rpm\n" $(SPECFILE)); \
-	if cd "$$tmp" && ! curl -f -L -O $(LATEST_SRPMS)/$$srpm; then \
-		echo "Failed to download $$srpm from latest Clear Linux release." >&2; \
-		echo "Run \"git pull\" and try again." >&2; \
+	if cd "$$tmp" && ! curl -s -S -f -L -O $(DOWNLOAD_MIRROR)/releases/$(1)/clear/source/SRPMS/$$srpm; then \
+		echo "Failed to download $$srpm from Clear Linux release $(1)." >&2; \
 		cd .. && rm -rf "$$tmp"; \
 		exit 1; \
 	fi; \
@@ -232,14 +218,58 @@ else
 	cd ..; \
 	srclist=$$(rpmspec --srpm -q --qf "[%{SOURCE}\n]" $(SPECFILE)); \
 	for s in $$srclist; do \
-		if ! mv "$$tmp"/$$s .; then \
+		if ! mv --no-clobber "$$tmp"/$$s .; then \
 			echo "Missing source file \"$$s\" in $${srpm}." >&2; \
 			rm -rf "$$tmp"; \
 			exit 1; \
 		fi; \
 		echo "Retrieved source file $$s ..."; \
 	done; \
-	rm -rf "$$tmp";
+	rm -rf "$$tmp"
+endef
+
+#help sources: If SOURCES_URL is defined, download required upstream source files
+#help from that location. Otherwise, try to download source files according to the
+#help URLs listed in the spec file. If any of the source files fail to download,
+#help check for the relevant SRPM from published releases of Clear Linux OS, and
+#help extract the files if found. Note that SRPMs are taken from the DOWNLOAD_MIRROR
+#help location. This will run automatically, as a dependency. NOTE: A Koji server can
+#help make use of this "make sources" command, since it lives in a repo named
+#help "common". If you use this makefile in Koji, ensure "make sources" continues to
+#help work for both remote and local builds, since it is a prerequisite of several
+#help commands for building packages.
+ifneq ($(strip $(SOURCES_URL)),)
+sources: upstream
+	while read u; do \
+		case "$$u" in \
+			"")    continue ;; \
+			*://*) n="$$u" ;; \
+			*)     n="$(SOURCES_URL)/$$u" ;; \
+		esac; \
+		if ! curl --fail -L "$$n" -o `basename "$$n"`; then \
+			echo "Failed to download $$n"; \
+			exit 1; \
+		fi; \
+	done < upstream
+else
+sources:
+	@$(MAKE) generateupstream; \
+	[ $$? -eq 0 ] && exit 0; \
+	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
+	for b in $$latest_builds; do \
+		url="$(DOWNLOAD_MIRROR)/releases/$$b/clear/source/package-sources"; \
+		echo "Checking for source files in build $$b ..."; \
+		if grep -q "$$nvr" <(curl -s -f -L $$url); then \
+			$(call fetch-from-srpm,$$b); \
+			if [ $$? -eq 0 ]; then \
+				echo "Source files retrieved from build $$b"; \
+				exit 0; \
+			fi; \
+		fi; \
+	done; \
+	echo "Source files not found for package"; \
+	exit 1
 endif
 
 prekoji-checks:
@@ -361,15 +391,20 @@ install:
 #help calculating their hashes. Autospec performs this step automatically, so ignore
 #help it for packages managed with autospec.
 generateupstream:
-	@rm -f upstream
-	@urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | grep Source | cut -d: -f2- | grep '://'); \
+	@[ -e upstream ] && mv upstream upstream.bak; \
+	urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | sed -n "s|^Source[0-9]*[[:blank:]]*:[[:blank:]]*\(..*://..*\)$$|\1|p"); \
 	for url in $$urls; do \
-	    filename=$$(basename $$url); \
-	    if [ ! -e $$filename ]; then \
-		curl --fail -L -O $$url; \
-	    fi; \
-	    echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
+		filename=$$(basename $$url); \
+		if [ ! -e $$filename ]; then \
+			if ! curl --fail -L -O $$url; then \
+				echo "Failed to download $$url"; \
+				[ -e upstream.bak ] && mv --no-clobber upstream.bak upstream; \
+				exit 1; \
+			fi; \
+		fi; \
+		echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
 	done
+	@rm -f upstream.bak
 	@cat upstream
 
 #help drop-abandoned: Remove all unused patches from the git tree

Makefile.shared

@@ -14,9 +14,9 @@ define loopup
 	@sudo partprobe /dev/loop$(DEVICE)
 	@sleep 1
 	@if [ -e /dev/loop$(DEVICE)p3 ]; then \
-	   sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
 	else \
-	   sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
 	fi
 	@sudo mount /dev/loop$(DEVICE)p1 $(TOPLVL)/image/boot
 endef
@@ -119,6 +119,24 @@ localrepostatus:
 		fi; \
 	)
 
+# Detection of whether to use the local RPM repo occurs on-the-fly. Normally,
+# variable definitions appear in Makefile.config, but these variables are not
+# intended to be user-modifiable.
+ifeq ($(wildcard $(TOPLVL)/repo/clear.cfg),)
+MOCK_CONFIG_VAL = $(realpath $(MOCK_CONF))
+else
+MOCK_CONFIG_VAL = $(realpath $(TOPLVL)/repo/clear.cfg)
+USE_LOCAL_REPO = 1
+endif
+
+localreponotice:
+	@if [ -n "${USE_LOCAL_REPO}" ]; then \
+		printf '\n\n** NOTICE: Using local repo for the build. Continuing after 3 seconds.\n'; \
+		printf '** If this is not desired, run `make repodisable`.\n'; \
+		printf '** The repo can be re-enabled later with `make repoenable`.\n'; \
+		printf '** Run `touch $(TOPLVL)/repo/opt-in-build` to prevent the 3 second delay.\n\n\n'; \
+		[ -f $(TOPLVL)/repo/opt-in-build ] || sleep 3; \
+	fi >&2
 
 .PHONY: help
 ##### Code

Makefile.toplevel

@@ -4,8 +4,8 @@ all: clone
 
 TOPLVL  = .
 
-include $(TOPLVL)/projects/common/Makefile.functions
 include $(TOPLVL)/projects/common/Makefile.config
+include $(TOPLVL)/projects/common/Makefile.shared
 
 SHELL   = /bin/bash
 
@@ -80,7 +80,11 @@ ${PULL_PKGS}:
 			fi ; \
 			git --no-pager log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit $$O..$$N; echo; \
 		fi ; \
-		$(call gitoliteurl,$$p); \
+		if [ "$$p" = "projects/autospec" ] ; then \
+			git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+		else \
+			$(call gitoliteurl,$$p); \
+		fi ; \
 	else \
 		echo "$$p: no such remote 'origin'"; \
 	fi ; \
@@ -188,7 +192,7 @@ provides:
 #help autospecnew: Creates a new autospec package with for a given URL=$(URL)
 #help with NAME=$(NAME). Several files used by autospec will be created in the
 #help process. For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
-autospecnew:
+autospecnew: localreponotice
 	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
@@ -204,7 +208,16 @@ autospecnew:
 			$(call subjectprefix,$(NAME)); \
 		); \
 		printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
-		python3 $(TOPLVL)/projects/autospec/autospec/autospec.py --integrity -t packages/$(NAME) --config $(AUTOSPEC_CONF) $(URL) --name $(NAME); \
+		python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
+			--target packages/$(NAME) \
+			--integrity \
+			--config "$(AUTOSPEC_CONF)" \
+			--name $(NAME) \
+			--archives $(ARCHIVES) \
+			--mock-config $(MOCK_CONFIG_VAL) \
+			$${SETVERSION:+ --version $${SETVERSION}} \
+			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+			$(URL); \
 		$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
 	else \
 		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \

user-setup.sh

@@ -149,4 +149,4 @@ fi
 echo -en "\n************************\n"
 
 echo 'Workspace has been set up in the "clearlinux" directory'
-echo 'NOTE: logout and log back in or run "newgrp kvm" to finalize the setup process'
+echo 'NOTE: logout and log back in to finalize the setup process'