Support newer versions of Mock (part 1)

In Mock version 1.4.4, the chroot group was changed from "mockbuild" to
"mock". This creates a problem for running Mock on Clear Linux because
(a) the distro already has group "mock", and (b) Mock tries to `groupdel
mock` within the chroot and fails, since the group is defined in
/usr/share/defaults/etc/group and not in /etc/group. Mock treats the
groupdel failure as fatal. Fortunately there is a workaround: set the
"chrootgroup" config to a group not already defined in the OS. I am
setting the group to "mockbuild" in the default config to restore the
behavior of previous Mock versions. Thanks to Mark Horn for adding this
feature upstream! It was merged for Mock 1.4.7.

Also, disable the new bootstrap container feature, added in Mock version
1.4.1. It was disabled by default in version 1.4.2, but keep the
explicit setting in case upstream enables it in the future.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

Makefile.common

@@ -12,16 +12,15 @@ SHELL  = /bin/bash
 SPECFILE = $(addsuffix .spec, $(PKG_NAME))
 
 SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || rpmspec -D '_vendor clr' --srpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' $(SPECFILE))
-SRPMFILE = results/$(SRPMVERS).src.rpm
+SRPMFILE = results/srpm/$(SRPMVERS).src.rpm
 
-LATEST_RPMS = $(shell [ ! -s results/build.log ] || for r in `egrep 'Wrote.*/RPMS/.*.rpm' results/build.log |  sed 's/.*\///'` ; do echo "results/$$r"; done)
+LATEST_RPMS = $(shell [ ! -s results/logs/build.log ] || for r in `egrep 'Wrote.*/RPMS/.*.rpm' results/logs/build.log |  sed 's/.*\///;s/.*debuginfo/debuginfo\/&/'` ; do echo "results/$$r"; done)
 RPMS ?= $(LATEST_RPMS)
 
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
 
 # MOCK_CONFIG_VAL is set in Makefile.shared
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL)
-
+MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_OPTS)
 
 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
@@ -89,20 +88,27 @@ prebuild-checks:
 #help files and rpms in the results directory of your package.  If there
 #help are mutlple tar.gz files for a package, make will do make sources,
 #help which creates a src.rpm file from the spec file.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options.
 build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
-	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
-	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
+	mkdir -p results/logs results/debuginfo
+	mv results/*.log results/logs
+	-mv results/*-debuginfo*.rpm results/debuginfo/
+	@perl $(TOPLVL)/projects/common/logcheck.pl results/logs/build.log
+	@perl $(TOPLVL)/projects/common/count.pl results/logs/build.log &> testresults
 	@$(MAKE) checkblacklist
 
 #help build-nocheck: Same as 'make build', but do not run the package's test suite.
 build-nocheck: configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --nocheck --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
-	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
-	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
+	mkdir -p results/logs results/debuginfo
+	mv results/*.log results/logs
+	-mv results/*-debuginfo*.rpm results/debuginfo/
+	@perl $(TOPLVL)/projects/common/logcheck.pl results/logs/build.log
+	@perl $(TOPLVL)/projects/common/count.pl results/logs/build.log &> testresults
 	@$(MAKE) checkblacklist
 
-fmvpatches: results/build.log
+fmvpatches: results/logs/build.log
 	@perl $(TOPLVL)/projects/common/fmvpatches.pl $(PKG_NAME)
 
 #help rootshell: Puts you in a root shell at the top of your build root.
@@ -117,7 +123,7 @@ shell:
 .PHONY: $(SRPMFILE)
 $(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice
 	@$(MAKE) spdxcheck
-	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/srpm/ --no-cleanup-after --uniqueext=$(PKG_NAME)
 
 # Do a git fetch and a git rebase to apply local commits on top of latest
 # commits from the remote. A git fetch/rebase is used rather than a git pull so
@@ -150,10 +156,12 @@ pullrebase:
 
 #help autospec: automatically generates a specfile. If there is
 #help already a specfile, it will be overwritten. Several files used by
-#help autospec will be created in the process. For more information, see
-#help the project at https://github.com/clearlinux/autospec
+#help autospec will be created in the process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: pullrebase localreponotice
-	rm -f results/*src.rpm
+	rm -f results/srpm/*src.rpm
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
@@ -166,6 +174,7 @@ autospec: pullrebase localreponotice
 		--name $(PKG_NAME) \
 		--archives $(ARCHIVES) \
 		--mock-config $(MOCK_CONFIG_VAL) \
+		--mock-opts "$(MOCK_OPTS)" \
 		$${SETVERSION:+ --version $${SETVERSION}} \
 		${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 		$(firstword $(NEWURL) $(URL));
@@ -205,30 +214,12 @@ bumpnogit:
 
 $(__missingsources): sources
 
-#help sources: If SOURCES_URL is defined, download required upstream source
-#help files from that location. Otherwise, try to extract source files from the
-#help SRPM in the latest Clear Linux release, as specified by the LATEST_SRPMS
-#help variable. This will run automatically, as a dependency. NOTE: A Koji
-#help server can make use of this "make sources" command, since it lives in a
-#help repo named "common". If you use this makefile in Koji, ensure
-#help "make sources" continues to work for both remote and local builds, since
-#help it is a prerequisite of several commands for building packages.
-sources: upstream
-ifneq ($(strip $(SOURCES_URL)),)
-	while read u; do \
-		case "$$u" in \
-		    "")    continue ;; \
-		    *://*) n="$$u" ;; \
-		    *)     n="$(SOURCES_URL)/$$u" ;; \
-		esac; \
-		curl --fail --noproxy "*" "$$n" -o `basename "$$n"`; \
-	done < upstream
-else
-	@tmp=$$(mktemp -d -p "."); \
+# First argument is a Clear Linux build number (e.g. 22000)
+define fetch-from-srpm
+	tmp=$$(mktemp -d -p "."); \
 	srpm=$$(rpmspec --srpm -q --qf "%{NVR}.src.rpm\n" $(SPECFILE)); \
-	if cd "$$tmp" && ! curl -f -L -O $(LATEST_SRPMS)/$$srpm; then \
-		echo "Failed to download $$srpm from latest Clear Linux release." >&2; \
-		echo "Run \"git pull\" and try again." >&2; \
+	if cd "$$tmp" && ! curl -s -S -f -L -O $(MIRROR_CURL_OPTS) $(DOWNLOAD_MIRROR)/releases/$(1)/clear/source/SRPMS/$$srpm; then \
+		echo "Failed to download $$srpm from Clear Linux release $(1)." >&2; \
 		cd .. && rm -rf "$$tmp"; \
 		exit 1; \
 	fi; \
@@ -236,14 +227,58 @@ else
 	cd ..; \
 	srclist=$$(rpmspec --srpm -q --qf "[%{SOURCE}\n]" $(SPECFILE)); \
 	for s in $$srclist; do \
-		if ! mv "$$tmp"/$$s .; then \
+		if ! mv --no-clobber "$$tmp"/$$s .; then \
 			echo "Missing source file \"$$s\" in $${srpm}." >&2; \
 			rm -rf "$$tmp"; \
 			exit 1; \
 		fi; \
 		echo "Retrieved source file $$s ..."; \
 	done; \
-	rm -rf "$$tmp";
+	rm -rf "$$tmp"
+endef
+
+#help sources: If SOURCES_URL is defined, download required upstream source files
+#help from that location. Otherwise, try to download source files according to the
+#help URLs listed in the spec file. If any of the source files fail to download,
+#help check for the relevant SRPM from published releases of Clear Linux OS, and
+#help extract the files if found. Note that SRPMs are taken from the DOWNLOAD_MIRROR
+#help location. This will run automatically, as a dependency. NOTE: A Koji server can
+#help make use of this "make sources" command, since it lives in a repo named
+#help "common". If you use this makefile in Koji, ensure "make sources" continues to
+#help work for both remote and local builds, since it is a prerequisite of several
+#help commands for building packages.
+ifneq ($(strip $(SOURCES_URL)),)
+sources: upstream
+	while read u; do \
+		case "$$u" in \
+			"")    continue ;; \
+			*://*) n="$$u" ;; \
+			*)     n="$(SOURCES_URL)/$$u" ;; \
+		esac; \
+		if ! curl --fail -L -o `basename "$$n"` $(SOURCES_CURL_OPTS) "$$n"; then \
+			echo "Failed to download $$n"; \
+			exit 1; \
+		fi; \
+	done < upstream
+else
+sources:
+	@$(MAKE) generateupstream; \
+	[ $$? -eq 0 ] && exit 0; \
+	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
+	for b in $$latest_builds; do \
+		url="$(DOWNLOAD_MIRROR)/releases/$$b/clear/source/package-sources"; \
+		echo "Checking for source files in build $$b ..."; \
+		if grep -q "$$nvr" <(curl -s -f -L $(MIRROR_CURL_OPTS) $$url); then \
+			$(call fetch-from-srpm,$$b); \
+			if [ $$? -eq 0 ]; then \
+				echo "Source files retrieved from build $$b"; \
+				exit 0; \
+			fi; \
+		fi; \
+	done; \
+	echo "Source files not found for package"; \
+	exit 1
 endif
 
 prekoji-checks:
@@ -257,8 +292,14 @@ koji: prekoji-checks kojidef
 	@if ! git branch | grep -q -E '^\* master'; then \
 		echo "Error: Must be on the master branch to submit to koji" >&2; \
 		exit 1; \
+	fi; \
+	if ! git diff --quiet HEAD ${SPECFILE}; then \
+		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
+		exit 1; \
+	fi; \
+	if git rev-parse --verify --quiet origin/master > /dev/null; then \
+		git pull --rebase; \
 	fi
-	git pull --rebase
 	git tag $(SRPMVERS)
 	git push origin master refs/tags/$(SRPMVERS)
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
@@ -286,7 +327,7 @@ rekoji-nowait:
 	$(MAKE) KOJI_NOWAIT="--nowait --background" rekoji
 
 logcheck:
-	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
+	@perl $(TOPLVL)/projects/common/logcheck.pl results/logs/build.log
 
 $(TOPLVL)/repo:
 	@echo "Creating local RPM repository $(TOPLVL)/repo"
@@ -296,8 +337,8 @@ $(TOPLVL)/repo:
 #help repoadd: Adds locally-built RPMs for this package to the local RPM
 #help repository. If this repository does not yet exist, it is created and
 #help enabled.
-repoadd: results/build.log $(TOPLVL)/repo
-	@if [ -z "`grep '^Wrote: .*$(shell basename $(SRPMFILE))$$' results/build.log`" ]; then \
+repoadd: results/logs/build.log $(TOPLVL)/repo
+	@if [ -z "`grep '^Wrote: .*$(shell basename $(SRPMFILE))$$' results/logs/build.log`" ]; then \
 		echo "You have not yet succesfully built this release yet"; exit 1; \
 	fi
 	@if [ -f .repo-index ]; then \
@@ -362,18 +403,25 @@ install:
 
 #help generateupstream: Run this rule to create or update the 'upstream' file
 #help by downloading the upstream source tarballs listed in the spec file and
-#help calculating their hashes. Autospec performs this step automatically, so ignore
-#help it for packages managed with autospec.
+#help calculating their hashes. Autospec performs this step automatically, so
+#help ignore it for packages managed with autospec. Additional curl options
+#help may be used for downloading the upstream source tarballs by defining the
+#help CURL_OPTS variable for this rule.
 generateupstream:
-	@rm -f upstream
-	@urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | grep Source | cut -d: -f2- | grep '://'); \
+	@[ -e upstream ] && mv upstream upstream.bak; \
+	urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | sed -n "s|^Source[0-9]*[[:blank:]]*:[[:blank:]]*\(..*://..*\)$$|\1|p"); \
 	for url in $$urls; do \
-	    filename=$$(basename $$url); \
-	    if [ ! -e $$filename ]; then \
-		curl --fail -L -O $$url; \
-	    fi; \
-	    echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
+		filename=$$(basename $$url); \
+		if [ ! -e $$filename ]; then \
+			if ! curl --fail -L -O $(CURL_OPTS) $$url; then \
+				echo "Failed to download $$url"; \
+				[ -e upstream.bak ] && mv --no-clobber upstream.bak upstream; \
+				exit 1; \
+			fi; \
+		fi; \
+		echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
 	done
+	@rm -f upstream.bak
 	@cat upstream
 
 #help drop-abandoned: Remove all unused patches from the git tree

Makefile.config

@@ -26,6 +26,9 @@ CGIT_BASE_URL =
 # Location of upstream tarball cache used by 'make build', 'make sources', etc.
 SOURCES_URL =
 
+# Additional curl options to use when downloading files from SOURCES_URL
+SOURCES_CURL_OPTS =
+
 # Location where package repos are hosted.
 PKG_BASE_URL = https://github.com/clearlinux-pkgs
 
@@ -38,6 +41,9 @@ PROJ_PKGS = autospec clr-bundles common
 # Clear Linux download mirror to use
 DOWNLOAD_MIRROR = https://cdn.download.clearlinux.org
 
+# Additional curl options to use when downloading files from DOWNLOAD_MIRROR
+MIRROR_CURL_OPTS =
+
 # Location of release content from latest Clear Linux release
 LATEST_RELEASE = $(DOWNLOAD_MIRROR)/current
 
@@ -48,17 +54,17 @@ LATEST_SRPMS = $(LATEST_RELEASE)/source/SRPMS
 PACKAGES_FILE = $(TOPLVL)/projects/common/packages
 
 # Configuration file for autospec.
-AUTOSPEC_CONF = $(TOPLVL)/projects/common/autospec.conf
+AUTOSPEC_CONF = $(TOPLVL)/projects/common/conf/autospec.conf
 
 # Set this variable to any non-empty value to use the package mappings stored
 # in "pkg-mapping" for cloning/pulling package repos.
 USE_PACKAGE_MAPPING = yes
 
-# Package manager config file. Default to the in-tree copy.
-PM_CONF = $(TOPLVL)/projects/common/image-creator/yum.conf
+# Package manager config file. Default to the in-tree copy of dnf.conf.
+PM_CONF = $(TOPLVL)/projects/common/conf/dnf.conf
 
 # Mock configuration file. Default to the in-tree copy.
-MOCK_CONF = $(TOPLVL)/projects/common/koji-client-files/clear.cfg
+MOCK_CONF = $(TOPLVL)/projects/common/conf/clear.cfg
 
 
 # Include any site local configuration variables. These variables may override

Makefile.shared

@@ -14,9 +14,9 @@ define loopup
 	@sudo partprobe /dev/loop$(DEVICE)
 	@sleep 1
 	@if [ -e /dev/loop$(DEVICE)p3 ]; then \
-	   sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
 	else \
-	   sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
 	fi
 	@sudo mount /dev/loop$(DEVICE)p1 $(TOPLVL)/image/boot
 endef

Makefile.toplevel

@@ -10,6 +10,7 @@ include $(TOPLVL)/projects/common/Makefile.shared
 SHELL   = /bin/bash
 
 PKGS    = $(shell cat $(PACKAGES_FILE))
+PKGS_LOCAL = $(sort $(filter ${PKGS},$(subst packages/,,$(wildcard packages/*))))
 
 proj_PKGS = $(addprefix proj_,$(PROJ_PKGS))
 clone_PKGS = $(addprefix clone_,$(PKGS))
@@ -64,7 +65,7 @@ cvecheck:
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
 .PHONY: pull ${PULL_PKGS} 
-PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,$(PKGS))
+PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,${PKGS_LOCAL})
 ${PULL_PKGS}:
 	@p=$(patsubst PULL_%,%,$@) ; \
 	if [ ! -d "$$p/.git" ]; then echo "Nothing to pull for $$p - $$p/.git missing"; exit 0; fi; \
@@ -191,12 +192,16 @@ provides:
 
 #help autospecnew: Creates a new autospec package with for a given URL=$(URL)
 #help with NAME=$(NAME). Several files used by autospec will be created in the
-#help process. For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
+#help process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
 autospecnew: localreponotice
 	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
+	-$(MAKE) clone_$(NAME)
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
@@ -215,6 +220,7 @@ autospecnew: localreponotice
 			--name $(NAME) \
 			--archives $(ARCHIVES) \
 			--mock-config $(MOCK_CONFIG_VAL) \
+			--mock-opts "$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
 			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 			$(URL); \

README.md

@@ -31,43 +31,13 @@ $ ./user-setup.sh
 After the script completes, make sure to logout and login again to complete the
 setup process.
 
-The script either accepts no options, or all (3) options in case you are
-configuring the Koji CLI for remote building on a Koji server. The options are
-documented in the script's `--help` output.
+The script accepts several options, or no options at all. The options are
+documented in the script's `--help` output. Note that if you are supplying any
+of the three Koji cert options (`-k`, `-s`, or `-c`), the other two options
+must be supplied as well.
 
-### Manual setup
-
-On your Clear Linux system, create a workspace for Clear Linux development
-work:
-
-```
-$ mkdir clearlinux
-```
-
-Clone this repo into a `projects` directory within the workspace:
-
-```
-$ cd clearlinux
-$ mkdir projects
-$ git clone https://github.com/clearlinux/common projects/common
-```
-
-Create the toplevel tooling Makefile:
-
-```
-$ ln -s projects/common/Makefile.toplevel Makefile
-```
-
-Clone all Clear Linux package and project repositories:
-
-```
-$ make clone
-```
-
-Note: You can clone the repos in parallel by using make's `-j` option.
-
-At this point, the `packages` directory will contain all Clear Linux package
-repos, and `projects` will contain common, clr-bundles, and autospec repos.
+If you do not wish to run the user-setup script, see the "Manual setup" section
+below for hints about how to initialize the tooling workspace.
 
 ## Example usage
 
@@ -141,3 +111,66 @@ package, a `make bump` command is available for this purpose.
 $ make bump
 $ make build
 ```
+
+## Other topics
+
+### Customizing the mock config
+
+In the past, the various make commands that call `mock` required a mock config
+installed at `/etc/mock/clear.cfg`. However, at present, the commands will
+instead use the mock config within this repo (`conf/clear.cfg`).
+
+If you wish to use a custom mock config, you must override the `MOCK_CONF`
+variable to specify a different value to pass to mock's `-r` option. The value
+is either a full path that ends with `.cfg`, or a config NAME installed at
+`/etc/mock/<NAME>.cfg`. You can override the `MOCK_CONF` config variable by
+redefining it in `Makefile.config.site_local`, which must reside at the
+toplevel directory in this repo.
+
+For example, to retain the old behavior of mock using `/etc/mock/clear.cfg`,
+add this line to `Makefile.config.site_local`:
+
+```
+MOCK_CONF = /etc/mock/clear.cfg
+```
+
+If `Makefile.config.site_local` doesn't exist already, create it.
+
+### Manual setup
+
+If you did not run the user-setup script (see "Automated setup" section above),
+you will want to set up the developer tooling workspace manually. This section
+provides general documentation for the manual setup process, and it is not
+meant to be exhaustive.
+
+On your Clear Linux system, create a workspace for Clear Linux development
+work:
+
+```
+$ mkdir clearlinux
+```
+
+Clone this repo into a `projects` directory within the workspace:
+
+```
+$ cd clearlinux
+$ mkdir projects
+$ git clone https://github.com/clearlinux/common projects/common
+```
+
+Create the toplevel tooling Makefile:
+
+```
+$ ln -s projects/common/Makefile.toplevel Makefile
+```
+
+Clone all Clear Linux package and project repositories:
+
+```
+$ make clone
+```
+
+Note: You can clone the repos in parallel by using make's `-j` option.
+
+At this point, the `packages` directory will contain all Clear Linux package
+repos, and `projects` will contain common, clr-bundles, and autospec repos.

conf/README.md

@@ -0,0 +1,31 @@
+# Configuration files
+
+This directory contains various configuration files used by the developer
+tooling framework.
+
+Unless otherwise noted in the config file documentation below, the config files
+can be modified with custom, site-local changes by modifying a variable in
+`Makefile.config.site_local` to point to a location of your choice (e.g.
+somewhere in `/etc`). The framework will then read the config file from the
+location you have specified instead, so copy the in-tree copy to that location
+as a starting point if you need to.
+
+* `autospec.conf`: default Autospec configuration file used by `make autospec`
+  and `make autospecnew`. All configuration is commented out by default, so
+  default Autospec settings are used. Config variable: `AUTOSPEC_CONF`
+
+* `clear.cfg`: default Mock configuration file used by various targets that
+  call out to Mock. Config variable: `MOCK_CONF`
+
+* `dnf.conf`: DNF conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `yum.conf`: YUM conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `koji.conf`: template Koji configuration file. It is installed by
+  `user-setup.sh` to `/etc/koji.conf`, and the in-tree copy is not used. If you
+  are using Koji with the framework, you will need to modify `/etc/koji.conf`
+  according to how the Koji instance is set up for your environment. Configuring
+  the installation location via `Makefile.config.site_local` is not possible at
+  the moment but is a planned feature.

conf/autospec.conf

@@ -18,6 +18,6 @@
 # an absolute path or a path relative to this autospec.conf.
 #packages_file =
 
-# Path to yum config, used for generating whatrequires. It can be an absolute
+# Path to dnf/yum config, used for generating whatrequires. It can be an absolute
 # path or a path relative to this autospec.conf.
 #yum_conf =

conf/clear.cfg

@@ -8,15 +8,18 @@ config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s
 config_opts['plugin_conf']['ccache_enable'] = False
 config_opts['releasever'] = 'clear'
 config_opts['package_manager'] = 'dnf'
+config_opts['chrootgroup'] = 'mockbuild'
+config_opts['use_bootstrap_container'] = False
 
 
-config_opts['yum.conf'] = """
+config_opts['dnf.conf'] = """
 
 [main]
-cachedir=/var/cache/yum
+cachedir=/var/cache/dnf
+keepcache=1
 debuglevel=1
 reposdir=/dev/null
-logfile=/var/log/yum.log
+logfile=/var/log/dnf.log
 retries=20
 obsoletes=1
 gpgcheck=0

conf/dnf.conf

@@ -0,0 +1,24 @@
+[main]
+keepcache=1
+debuglevel=1
+logfile=/var/log/dnf.log
+exactarch=1
+obsoletes=1
+gpgcheck=0
+plugins=0
+retries=10
+installonly_limit=3
+reposdir=/root/mash
+
+[clear]
+name=Clear
+baseurl=https://cdn.download.clearlinux.org/current/x86_64/os/
+enabled=1
+gpgcheck=0
+
+[clear-source]
+name=Clear sources
+failovermethod=priority
+baseurl=https://cdn.download.clearlinux.org/current/source/SRPMS/
+enabled=1
+gpgcheck=0

fmvpatches.pl

@@ -82,7 +82,7 @@ sub find_file {
     return $file;
 }
 
-open(BUILD_LOG, '<', "results/build.log") or die $!;
+open(BUILD_LOG, '<', "results/logs/build.log") or die $!;
 while (<BUILD_LOG>) {
     if($_ =~ /(\S+):([0-9]*):([0-9]*): note: (basic block|loop) (vectorized)/) {
 	$fmv->{s_name} = (split('/',$1))[-1];

provides.sh

@@ -23,7 +23,7 @@ if [ "$#" -eq 0 ]; then
 fi
 
 if [ ! -e /etc/yum.conf ]; then
-  echo "Error: yum.conf is missing. Please copy projects/common/image-creator/yum.conf to /etc"
+  echo "Error: yum.conf is missing. Please copy projects/common/conf/yum.conf to /etc"
   exit 1
 fi
 

start_qemu.sh

@@ -54,4 +54,5 @@ qemu-system-x86_64 \
     -drive file="$IMAGE",if=virtio,aio=threads,format=raw \
     -netdev user,id=mynet0,hostfwd=tcp::${VMN}0022-:22,hostfwd=tcp::${VMN}2375-:2375 \
     -device virtio-net-pci,netdev=mynet0 \
+    -device virtio-rng-pci \
     -debugcon file:debug.log -global isa-debugcon.iobase=0x402 $@

user-setup.sh

@@ -1,102 +1,131 @@
-#!/bin/sh
+#!/bin/bash
 
 SCRIPT=$(/usr/bin/basename $0)
 PEM=""
 SERVERCA=""
 CLIENTCA=""
+WORKSPACE="clearlinux"
+PACKAGE_REPOS=
 
 help() {
-	printf "%s\n" >&2 "Usage: $SCRIPT [options]" \
-			  "" \
-			  "Options:" \
-			  "-k --client-cert PEM_FILE: Enable client user cert for koji configuration; requires a PEM file argument" \
-			  "-s --server-ca PEM_FILE: Enable server CA cert for koji configuration; requires a PEM file argument" \
-			  "-c --client-ca PEM_FILE: Enable client CA cert for koji configuration; requires a PEM file argument" \
-			  ""
+  printf "%s\n" >&2 "Usage: $SCRIPT [options]" \
+    "" \
+    "Options:" \
+    "" \
+    "-d --directory NAME: Set up workspace in the given directory." \
+    "-a --clone-packages: Clone all package repos." \
+    "-j --jobs [NUM]: Clone repos with NUM jobs. If NUM is not given, it is set to the available CPU count." \
+    "" \
+    "-k --client-cert PEM_FILE: Enable client user cert for koji configuration; requires a PEM file argument" \
+    "-s --server-ca PEM_FILE: Enable server CA cert for koji configuration; requires a PEM file argument" \
+    "-c --client-ca PEM_FILE: Enable client CA cert for koji configuration; requires a PEM file argument" \
+    ""
+}
+
+error() {
+  echo -e "Error: $1\n" >&2
+  help
+  exit 1
 }
 
 while [ $# -gt 0 ]; do
-	case "$1" in
-		"--help"|"-h")
-			help
-			exit 0
-			;;
-		"--client-cert"|"-k")
-			shift
-			PEM="$PWD/$1"
-			;;
-		"--server-ca"|"-s")
-			shift
-			SERVERCA="$PWD/$1"
-			;;
-		"--client-ca"|"-c")
-			shift
-			CLIENTCA="$PWD/$1"
-			;;
-		*)
-			help
-			exit 1
-			;;
-	esac
-	shift
+  case "$1" in
+    "--help"|"-h")
+      help
+      exit 0
+      ;;
+    "--client-cert"|"-k")
+      shift
+      PEM="$(realpath $1)"
+      ;;
+    "--server-ca"|"-s")
+      shift
+      SERVERCA="$(realpath $1)"
+      ;;
+    "--client-ca"|"-c")
+      shift
+      CLIENTCA="$(realpath $1)"
+      ;;
+    "--jobs"|"-j")
+      if echo "$2" | grep -qx "[1-9][0-9]*"; then
+        shift
+        JOBS="$1"
+      elif [ -f /proc/cpuinfo ]; then
+        JOBS=$(grep -Ec '^processor.*:.*[0-9]+$' /proc/cpuinfo)
+      fi
+      ;;
+    "--directory"|"-d")
+      [ -z "$2" ] && error "Must supply a directory name to the -d option"
+      [ "${2:0:1}" = "-" ] && error "Directory name cannot begin with \"-\""
+      shift
+      WORKSPACE="$1"
+      ;;
+    "--clone-packages"|"-a")
+      PACKAGE_REPOS=1
+      ;;
+    *)
+      help
+      exit 1
+      ;;
+  esac
+  shift
 done
 
-error() {
-	echo -e "Error: $1\n" >&2
-	help
-	exit 1
-}
 
 if [ -z "$PEM" ] && [ -z "$SERVERCA" ] && [ -z "$CLIENTCA" ]; then
-	USE_KOJI=
+  USE_KOJI=
 else
-	if [ -z "$PEM" ] || [ -z "$SERVERCA" ] || [ -z "$CLIENTCA" ]; then
-		error "Must specify all three command line options (or none)"
-	fi
-	if [ ! -f "$PEM" ]; then
-		error "Missing koji client PEM key file"
-	fi
-	if [ ! -f "$SERVERCA" ]; then
-		error "Missing koji server CA PEM file"
-	fi
-	if [ ! -f "$CLIENTCA" ]; then
-		error "Missing koji client CA PEM file"
-	fi
-	USE_KOJI="yes"
+  if [ -z "$PEM" ] || [ -z "$SERVERCA" ] || [ -z "$CLIENTCA" ]; then
+    error "Must specify all three command line options (or none)"
+  fi
+  if [ ! -f "$PEM" ]; then
+    error "Missing koji client PEM key file"
+  fi
+  if [ ! -f "$SERVERCA" ]; then
+    error "Missing koji server CA PEM file"
+  fi
+  if [ ! -f "$CLIENTCA" ]; then
+    error "Missing koji client CA PEM file"
+  fi
+  USE_KOJI="yes"
+fi
+
+if [ -n "$JOBS" ]; then
+  JOBS_ARG="-j $JOBS"
+fi
+
+if [ -d "$WORKSPACE" ]; then
+  error "Directory \"$WORKSPACE\" already exists. \
+Either remove this workspace, or use a different workspace name."
 fi
 
 required_progs() {
-	local bindir="/usr/bin"
-	for f in git mock rpm rpmbuild ; do
-		[ ! -x "${bindir}/${f}" ] && missing+="${f} "
-	done
-	[ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
-	if [ -n "$missing" ]; then
-		echo "Install the following programs and re-run this script:" >&2
-		echo $missing >&2
-		echo 'All programs should be provided in the "os-clr-on-clr" bundle.' >&2
-		exit 1
-	fi
+  local bindir="/usr/bin"
+  for f in git mock rpm rpmbuild ; do
+    [ ! -x "${bindir}/${f}" ] && missing+="${f} "
+  done
+  [ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
+  if [ -n "$missing" ]; then
+    echo "Install the following programs and re-run this script:" >&2
+    echo $missing >&2
+    echo 'All programs should be provided in the "os-clr-on-clr" bundle.' >&2
+    exit 1
+  fi
 }
 
 required_progs
 
-echo 'Initializing development workspace in "clearlinux" . . .'
-if [ -d "clearlinux" ]; then
-	echo 'Directory "clearlinux" already exists in current directory.' >&2
-	echo "Cannot initialize workspace." >&2
-	exit 1
-fi
+echo "Initializing development workspace in \"$WORKSPACE\" . . ."
 
-mkdir clearlinux
-cd clearlinux
+mkdir "$WORKSPACE"
+cd "$WORKSPACE"
 
 echo "Setting up common repo . . ."
 mkdir projects
 git clone https://github.com/clearlinux/common projects/common
 if [ $? -ne 0 ]; then
-	echo "Failed to clone common repo." >&2
-	exit 1
+  echo "Failed to clone common repo." >&2
+  exit 1
 fi
 
 # Finish setup for packages/projects hierarchy
@@ -105,48 +134,52 @@ mkdir -p packages/common
 ln -sf ../../projects/common/Makefile.common packages/common/Makefile.common
 
 if [ "$USE_KOJI" ]; then
-	echo "Setting up koji certs . . ."
-	mkdir -p ~/.koji
-	cp "$PEM" ~/.koji/client.crt
-	cp "$CLIENTCA" ~/.koji/clientca.crt
-	cp "$SERVERCA" ~/.koji/serverca.crt
+  echo "Setting up koji certs . . ."
+  mkdir -p ~/.koji
+  cp "$PEM" ~/.koji/client.crt
+  cp "$CLIENTCA" ~/.koji/clientca.crt
+  cp "$SERVERCA" ~/.koji/serverca.crt
 
-	if [ ! -f /etc/koji.conf ]; then
-		echo "Setting up koji config . . ."
-		sudo cp projects/common/koji-client-files/koji.conf /etc
-	fi
-fi
-
-if [ ! -f /etc/mock/clear.cfg ]; then
-	echo "Setting up mock config . . ."
-	sudo mkdir -p /etc/mock
-	sudo cp projects/common/koji-client-files/clear.cfg /etc/mock
+  if [ ! -f /etc/koji.conf ]; then
+    echo "Setting up koji config . . ."
+    sudo cp projects/common/conf/koji.conf /etc
+  fi
 fi
 
 echo "Adding user to kvm group . . ."
 sudo usermod -a -G kvm $USER
 
 echo "Cloning special project repositories . . ."
-make clone-projects
+make ${JOBS_ARG} clone-projects
 
-if [ -z "$NO_PACKAGE_REPOS" ]; then
-	echo "Cloning all package repositories . . ."
-	make clone-packages
+if [ -n "$PACKAGE_REPOS" ]; then
+  echo "Cloning all package repositories . . ."
+  make ${JOBS_ARG} clone-packages
 fi
 
+echo "Creating mix workspace . . ."
+mkdir -p mix
+
 if [ "$USE_KOJI" ]; then
-	echo "Testing koji installation . . ."
-	if koji moshimoshi; then
-		echo -en "\n************************\n\n"
-		echo "Koji installed and configured successfully"
-	else
-		echo -en "\n************************\n\n"
-		echo "Error with koji installation or configuration" >&2
-		exit 1
-	fi
+  echo "Testing koji installation . . ."
+  if koji moshimoshi; then
+    echo -en "\n************************\n\n"
+    echo "Koji installed and configured successfully"
+  else
+    echo -en "\n************************\n\n"
+    echo "Error with koji installation or configuration" >&2
+    exit 1
+  fi
 fi
 
 echo -en "\n************************\n"
 
-echo 'Workspace has been set up in the "clearlinux" directory'
+echo "Workspace has been set up in \"$WORKSPACE\""
+if [ -z "$PACKAGE_REPOS" ]; then
+  echo "NOTE: To clone all package repos, run \"cd $WORKSPACE; make [-j NUM] clone-packages\""
+  echo "NOTE: To clone a single package repo with NAME, run \"cd $WORKSPACE; make clone_NAME\""
+fi
 echo 'NOTE: logout and log back in to finalize the setup process'
+
+
+# vi: ft=sh sw=2 et sts=2