koji: exit early if spec has uncommitted changes

When running 'make koji', the tooling will attempt to create a new git
tag with a name constructed from the spec file's NVR.

However, if there are uncommitted changes to the Name, Version, or
Release fields (collectively, NVR) in that file, the new tag's name will
not match the NVR in the spec file referenced by commit HEAD.

Protect against the mismatch by exiting early from a 'make koji' if any
spec file modifications are not yet committed to the repo, and alert the
user what the problem is.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

Makefile.common

@@ -12,9 +12,9 @@ SHELL  = /bin/bash
 SPECFILE = $(addsuffix .spec, $(PKG_NAME))
 
 SRPMVERS = $(shell [ ! -f $(SPECFILE) ] || rpmspec -D '_vendor clr' --srpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' $(SPECFILE))
-SRPMFILE = results/$(SRPMVERS).src.rpm
+SRPMFILE = results/srpm/$(SRPMVERS).src.rpm
 
-LATEST_RPMS = $(shell [ ! -s results/build.log ] || for r in `egrep 'Wrote.*/RPMS/.*.rpm' results/build.log |  sed 's/.*\///'` ; do echo "results/$$r"; done)
+LATEST_RPMS = $(shell [ ! -s results/logs/build.log ] || for r in `egrep 'Wrote.*/RPMS/.*.rpm' results/logs/build.log |  sed 's/.*\///;s/.*debuginfo/debuginfo\/&/'` ; do echo "results/$$r"; done)
 RPMS ?= $(LATEST_RPMS)
 
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
@@ -91,18 +91,24 @@ prebuild-checks:
 #help Use MOCK_OPTS environment varible to pass down arbitrary mock options.
 build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
-	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
-	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
+	mkdir -p results/logs results/debuginfo
+	mv results/*.log results/logs
+	mv results/*-debuginfo*.rpm results/debuginfo/
+	@perl $(TOPLVL)/projects/common/logcheck.pl results/logs/build.log
+	@perl $(TOPLVL)/projects/common/count.pl results/logs/build.log &> testresults
 	@$(MAKE) checkblacklist
 
 #help build-nocheck: Same as 'make build', but do not run the package's test suite.
 build-nocheck: configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --nocheck --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
-	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
-	@perl $(TOPLVL)/projects/common/count.pl results/build.log &> testresults
+	mkdir -p results/logs results/debuginfo
+	mv results/*.log results/logs
+	mv results/*-debuginfo*.rpm results/debuginfo/
+	@perl $(TOPLVL)/projects/common/logcheck.pl results/logs/build.log
+	@perl $(TOPLVL)/projects/common/count.pl results/logs/build.log &> testresults
 	@$(MAKE) checkblacklist
 
-fmvpatches: results/build.log
+fmvpatches: results/logs/build.log
 	@perl $(TOPLVL)/projects/common/fmvpatches.pl $(PKG_NAME)
 
 #help rootshell: Puts you in a root shell at the top of your build root.
@@ -117,7 +123,7 @@ shell:
 .PHONY: $(SRPMFILE)
 $(SRPMFILE): $(SPECFILE) $(__allsources) localreponotice
 	@$(MAKE) spdxcheck
-	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
+	$(MOCK) --buildsrpm --source=./ --spec=$(SPECFILE) --result=results/srpm/ --no-cleanup-after --uniqueext=$(PKG_NAME)
 
 # Do a git fetch and a git rebase to apply local commits on top of latest
 # commits from the remote. A git fetch/rebase is used rather than a git pull so
@@ -155,7 +161,7 @@ pullrebase:
 #help to autospec.
 #help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: pullrebase localreponotice
-	rm -f results/*src.rpm
+	rm -f results/srpm/*src.rpm
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
@@ -286,8 +292,14 @@ koji: prekoji-checks kojidef
 	@if ! git branch | grep -q -E '^\* master'; then \
 		echo "Error: Must be on the master branch to submit to koji" >&2; \
 		exit 1; \
+	fi; \
+	if ! git diff --quiet HEAD ${SPECFILE}; then \
+		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
+		exit 1; \
+	fi; \
+	if git rev-parse --verify --quiet origin/master > /dev/null; then \
+		git pull --rebase; \
 	fi
-	git pull --rebase
 	git tag $(SRPMVERS)
 	git push origin master refs/tags/$(SRPMVERS)
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
@@ -315,7 +327,7 @@ rekoji-nowait:
 	$(MAKE) KOJI_NOWAIT="--nowait --background" rekoji
 
 logcheck:
-	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
+	@perl $(TOPLVL)/projects/common/logcheck.pl results/logs/build.log
 
 $(TOPLVL)/repo:
 	@echo "Creating local RPM repository $(TOPLVL)/repo"
@@ -325,8 +337,8 @@ $(TOPLVL)/repo:
 #help repoadd: Adds locally-built RPMs for this package to the local RPM
 #help repository. If this repository does not yet exist, it is created and
 #help enabled.
-repoadd: results/build.log $(TOPLVL)/repo
-	@if [ -z "`grep '^Wrote: .*$(shell basename $(SRPMFILE))$$' results/build.log`" ]; then \
+repoadd: results/logs/build.log $(TOPLVL)/repo
+	@if [ -z "`grep '^Wrote: .*$(shell basename $(SRPMFILE))$$' results/logs/build.log`" ]; then \
 		echo "You have not yet succesfully built this release yet"; exit 1; \
 	fi
 	@if [ -f .repo-index ]; then \

Makefile.config

@@ -54,17 +54,17 @@ LATEST_SRPMS = $(LATEST_RELEASE)/source/SRPMS
 PACKAGES_FILE = $(TOPLVL)/projects/common/packages
 
 # Configuration file for autospec.
-AUTOSPEC_CONF = $(TOPLVL)/projects/common/autospec.conf
+AUTOSPEC_CONF = $(TOPLVL)/projects/common/conf/autospec.conf
 
 # Set this variable to any non-empty value to use the package mappings stored
 # in "pkg-mapping" for cloning/pulling package repos.
 USE_PACKAGE_MAPPING = yes
 
-# Package manager config file. Default to the in-tree copy.
-PM_CONF = $(TOPLVL)/projects/common/image-creator/yum.conf
+# Package manager config file. Default to the in-tree copy of dnf.conf.
+PM_CONF = $(TOPLVL)/projects/common/conf/dnf.conf
 
 # Mock configuration file. Default to the in-tree copy.
-MOCK_CONF = $(TOPLVL)/projects/common/koji-client-files/clear.cfg
+MOCK_CONF = $(TOPLVL)/projects/common/conf/clear.cfg
 
 
 # Include any site local configuration variables. These variables may override

Makefile.toplevel

@@ -10,6 +10,7 @@ include $(TOPLVL)/projects/common/Makefile.shared
 SHELL   = /bin/bash
 
 PKGS    = $(shell cat $(PACKAGES_FILE))
+PKGS_LOCAL = $(sort $(filter ${PKGS},$(subst packages/,,$(wildcard packages/*))))
 
 proj_PKGS = $(addprefix proj_,$(PROJ_PKGS))
 clone_PKGS = $(addprefix clone_,$(PKGS))
@@ -64,7 +65,7 @@ cvecheck:
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
 .PHONY: pull ${PULL_PKGS} 
-PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,$(PKGS))
+PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,${PKGS_LOCAL})
 ${PULL_PKGS}:
 	@p=$(patsubst PULL_%,%,$@) ; \
 	if [ ! -d "$$p/.git" ]; then echo "Nothing to pull for $$p - $$p/.git missing"; exit 0; fi; \

README.md

@@ -114,6 +114,28 @@ $ make build
 
 ## Other topics
 
+### Customizing the mock config
+
+In the past, the various make commands that call `mock` required a mock config
+installed at `/etc/mock/clear.cfg`. However, at present, the commands will
+instead use the mock config within this repo (`conf/clear.cfg`).
+
+If you wish to use a custom mock config, you must override the `MOCK_CONF`
+variable to specify a different value to pass to mock's `-r` option. The value
+is either a full path that ends with `.cfg`, or a config NAME installed at
+`/etc/mock/<NAME>.cfg`. You can override the `MOCK_CONF` config variable by
+redefining it in `Makefile.config.site_local`, which must reside at the
+toplevel directory in this repo.
+
+For example, to retain the old behavior of mock using `/etc/mock/clear.cfg`,
+add this line to `Makefile.config.site_local`:
+
+```
+MOCK_CONF = /etc/mock/clear.cfg
+```
+
+If `Makefile.config.site_local` doesn't exist already, create it.
+
 ### Manual setup
 
 If you did not run the user-setup script (see "Automated setup" section above),

conf/README.md

@@ -0,0 +1,31 @@
+# Configuration files
+
+This directory contains various configuration files used by the developer
+tooling framework.
+
+Unless otherwise noted in the config file documentation below, the config files
+can be modified with custom, site-local changes by modifying a variable in
+`Makefile.config.site_local` to point to a location of your choice (e.g.
+somewhere in `/etc`). The framework will then read the config file from the
+location you have specified instead, so copy the in-tree copy to that location
+as a starting point if you need to.
+
+* `autospec.conf`: default Autospec configuration file used by `make autospec`
+  and `make autospecnew`. All configuration is commented out by default, so
+  default Autospec settings are used. Config variable: `AUTOSPEC_CONF`
+
+* `clear.cfg`: default Mock configuration file used by various targets that
+  call out to Mock. Config variable: `MOCK_CONF`
+
+* `dnf.conf`: DNF conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `yum.conf`: YUM conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `koji.conf`: template Koji configuration file. It is installed by
+  `user-setup.sh` to `/etc/koji.conf`, and the in-tree copy is not used. If you
+  are using Koji with the framework, you will need to modify `/etc/koji.conf`
+  according to how the Koji instance is set up for your environment. Configuring
+  the installation location via `Makefile.config.site_local` is not possible at
+  the moment but is a planned feature.

conf/autospec.conf

@@ -18,6 +18,6 @@
 # an absolute path or a path relative to this autospec.conf.
 #packages_file =
 
-# Path to yum config, used for generating whatrequires. It can be an absolute
+# Path to dnf/yum config, used for generating whatrequires. It can be an absolute
 # path or a path relative to this autospec.conf.
 #yum_conf =

conf/clear.cfg

@@ -10,13 +10,14 @@ config_opts['releasever'] = 'clear'
 config_opts['package_manager'] = 'dnf'
 
 
-config_opts['yum.conf'] = """
+config_opts['dnf.conf'] = """
 
 [main]
-cachedir=/var/cache/yum
+cachedir=/var/cache/dnf
+keepcache=1
 debuglevel=1
 reposdir=/dev/null
-logfile=/var/log/yum.log
+logfile=/var/log/dnf.log
 retries=20
 obsoletes=1
 gpgcheck=0

conf/dnf.conf

@@ -0,0 +1,24 @@
+[main]
+keepcache=1
+debuglevel=1
+logfile=/var/log/dnf.log
+exactarch=1
+obsoletes=1
+gpgcheck=0
+plugins=0
+retries=10
+installonly_limit=3
+reposdir=/root/mash
+
+[clear]
+name=Clear
+baseurl=https://cdn.download.clearlinux.org/current/x86_64/os/
+enabled=1
+gpgcheck=0
+
+[clear-source]
+name=Clear sources
+failovermethod=priority
+baseurl=https://cdn.download.clearlinux.org/current/source/SRPMS/
+enabled=1
+gpgcheck=0

fmvpatches.pl

@@ -82,7 +82,7 @@ sub find_file {
     return $file;
 }
 
-open(BUILD_LOG, '<', "results/build.log") or die $!;
+open(BUILD_LOG, '<', "results/logs/build.log") or die $!;
 while (<BUILD_LOG>) {
     if($_ =~ /(\S+):([0-9]*):([0-9]*): note: (basic block|loop) (vectorized)/) {
 	$fmv->{s_name} = (split('/',$1))[-1];

packages

@@ -7,6 +7,8 @@ FreeRDP
 GConf
 Jinja2
 Keras
+Keras_Applications
+Keras_Preprocessing
 LS_COLORS
 LVM2
 LibRaw
@@ -25,6 +27,7 @@ OpenSP
 Paste
 PasteDeploy
 Pillow
+PyMySQL
 PyWavelets
 PyYAML
 Pygments
@@ -222,9 +225,11 @@ R-flexmix
 R-forcats
 R-foreach
 R-formatR
+R-fossil
 R-fpc
 R-futile.logger
 R-futile.options
+R-future
 R-gam
 R-gamm4
 R-gbm
@@ -239,6 +244,7 @@ R-ggplot2movies
 R-ggridges
 R-git2r
 R-glmnet
+R-globals
 R-glue
 R-gmodels
 R-gmp
@@ -277,6 +283,7 @@ R-labelled
 R-laeken
 R-lambda.r
 R-lars
+R-later
 R-latticeExtra
 R-lava
 R-lazyeval
@@ -285,6 +292,7 @@ R-leaflet
 R-leaps
 R-lfe
 R-libcoin
+R-listenv
 R-lle
 R-lme4
 R-lmtest
@@ -374,6 +382,7 @@ R-processx
 R-prodlim
 R-profileModel
 R-progress
+R-promises
 R-proto
 R-proxy
 R-pryr
@@ -432,6 +441,7 @@ R-setRNG
 R-sfsmisc
 R-sgeostat
 R-shape
+R-shapefiles
 R-shiny
 R-shinyjs
 R-shinystan
@@ -476,6 +486,7 @@ R-tidyr
 R-tidyselect
 R-tidyverse
 R-timeDate
+R-tinytex
 R-tis
 R-tm
 R-topicmodels
@@ -500,6 +511,7 @@ R-xtable
 R-xts
 R-yaml
 R-zCompositions
+R-zip
 R-zoo
 Remmina
 SDL
@@ -536,7 +548,6 @@ accountsservice
 acl
 acme
 acpica-unix2
-acrn-devicemodel
 acrn-hypervisor
 adwaita-icon-theme
 aeon
@@ -702,6 +713,7 @@ colord
 colord-gtk
 comedilib
 compat-SDL2-soname1
+compat-cryptsetup-soname4
 compat-efivar-soname0
 compat-fuse-soname2
 compat-gegl
@@ -729,6 +741,7 @@ connect-proxy
 connman
 connman-gtk
 conntrack-tools
+console-autostart
 containerd
 contextlib2
 coreutils
@@ -755,6 +768,7 @@ cups-filters
 cups-pk-helper
 curl
 cve-check-tool
+cxxfilt
 cycler
 dapl
 darktable
@@ -763,6 +777,7 @@ db
 dbus
 dbus-glib
 dconf
+dconf-editor
 deap
 decorator
 defusedxml
@@ -776,6 +791,7 @@ diffoscope
 diffstat
 diffutils
 dist-pam-configs
+distro
 dmenu
 dmidecode
 dnf
@@ -894,6 +910,7 @@ gevent
 gexiv2
 gfbgraph
 gflags
+gftp
 ghostscript
 gimp
 git
@@ -905,6 +922,7 @@ glib
 glib-networking
 glibc
 glibmm
+glmark2
 global
 glog
 glslang
@@ -1031,6 +1049,7 @@ i2c-tools
 i3
 i3lock
 i3status
+iasimage
 ibus
 icalendar
 icdiff
@@ -1100,6 +1119,11 @@ jupyter_core
 jupyterlab
 jupyterlab_launcher
 kafka-dep
+kata-image
+kata-proxy
+kata-qemu-lite
+kata-runtime
+kata-shim
 kbd
 kernel-install
 kexec-tools
@@ -1205,6 +1229,7 @@ libisofs
 libjpeg-turbo
 libjpeg-turbo-soname8
 libksba
+libmbim
 libmediaart
 libmnl
 libmspack
@@ -1230,6 +1255,7 @@ libpipeline
 libpng
 libpthread-stubs
 libpwquality
+libqmi
 librepo
 librsvg
 librtlsdr
@@ -1258,6 +1284,7 @@ libunistring
 libunwind
 libusb
 libuser
+libuv
 libva
 libva-intel-driver
 libva-utils
@@ -1283,6 +1310,7 @@ libxslt
 libyami
 libyami-utils
 libzip
+libzmq
 linecache2
 links
 linux
@@ -1293,6 +1321,7 @@ linux-gce
 linux-hyperv
 linux-hyperv-lts
 linux-hyperv-mini
+linux-kata
 linux-kvm
 linux-libc-headers
 linux-lts
@@ -1329,13 +1358,17 @@ mc
 mccabe
 mcelog
 mdadm
+meld
+memkind
 memshrinker
 mesa
 mesa-demos
 meson
 metacity
+metrics-discovery
 micro-config-drive
 micro-config-drive-aws
+micro-config-drive-oci
 mido
 minetest
 minetest_game
@@ -1345,10 +1378,12 @@ mixer-tools
 mkfontdir
 mkfontscale
 mkosi
+mobile-broadband-provider-info
 mock
 mod_wsgi
 moka-icon-theme
 more-itertools
+moreutils
 mosh
 motd-update
 motif
@@ -1358,6 +1393,7 @@ mozjs52
 mpc
 mpfr
 mpg123
+mpi4py
 msgpack-python
 msmtp
 msr-tools
@@ -1423,6 +1459,7 @@ numpydoc
 nvme-cli
 oath-toolkit
 oauth2client
+ocaml
 ocl-icd
 offlineimap
 olefile
@@ -1474,6 +1511,7 @@ patch
 patchutils
 pavucontrol
 pbr
+pbr-legacy
 pbzip2
 pciutils
 pcre
@@ -1570,6 +1608,7 @@ perl-Test-NoWarnings
 perl-Test-Output
 perl-Test-Pod
 perl-Test-Requires
+perl-Test-RequiresInternet
 perl-Test-Simple
 perl-Test-Taint
 perl-Test-Warn
@@ -1604,6 +1643,7 @@ pidgin-sipe
 pigz
 pinentry
 pip
+pip-legacy
 pixman
 pkcs11-helper
 pkg-config
@@ -1670,6 +1710,7 @@ python-augeas
 python-dateutil
 python-future
 python-gflags
+python-kconfiglib
 python-krbV
 python-magic
 python-memcached
@@ -1679,6 +1720,7 @@ python-multilib
 python-pkgconfig
 python-pocketlint
 python-polib
+python-rpm-legacypython
 python-subunit
 python-systemd
 python-tappy
@@ -1743,6 +1785,7 @@ rpm
 rr
 rsa
 rsync
+rt-tests
 ruby
 runc
 rust-std32
@@ -1773,6 +1816,7 @@ serf
 setproctitle
 setserial
 setuptools
+setuptools-legacy
 setuptools_scm
 setuptools_scm_git_archive
 shadow
@@ -1811,11 +1855,13 @@ spice-protocol
 spyder-notebook
 sqlite-autoconf
 sqlparse
+squashfs-tools
 sshfs
 stack
 startup-notification
 stow
 strace
+stress-ng
 subunit
 subversion
 sudo
@@ -1853,6 +1899,7 @@ testresources
 testscenarios
 testtools
 texinfo
+texlive
 tflearn
 thermal_daemon
 thin-provisioning-tools
@@ -1888,6 +1935,7 @@ uhttpmock
 unbundle
 unicode-xid
 unifdef
+unison
 unittest2
 unzip
 upower
@@ -1918,6 +1966,7 @@ volume_key
 vsqlite
 vte
 vulkan-sdk
+waffle
 waitress
 wayland
 wayland-protocols

provides.sh

@@ -23,7 +23,7 @@ if [ "$#" -eq 0 ]; then
 fi
 
 if [ ! -e /etc/yum.conf ]; then
-  echo "Error: yum.conf is missing. Please copy projects/common/image-creator/yum.conf to /etc"
+  echo "Error: yum.conf is missing. Please copy projects/common/conf/yum.conf to /etc"
   exit 1
 fi
 

start_qemu.sh

@@ -54,4 +54,5 @@ qemu-system-x86_64 \
     -drive file="$IMAGE",if=virtio,aio=threads,format=raw \
     -netdev user,id=mynet0,hostfwd=tcp::${VMN}0022-:22,hostfwd=tcp::${VMN}2375-:2375 \
     -device virtio-net-pci,netdev=mynet0 \
+    -device virtio-rng-pci \
     -debugcon file:debug.log -global isa-debugcon.iobase=0x402 $@

user-setup.sh

@@ -36,15 +36,15 @@ while [ $# -gt 0 ]; do
       ;;
     "--client-cert"|"-k")
       shift
-      PEM="$PWD/$1"
+      PEM="$(realpath $1)"
       ;;
     "--server-ca"|"-s")
       shift
-      SERVERCA="$PWD/$1"
+      SERVERCA="$(realpath $1)"
       ;;
     "--client-ca"|"-c")
       shift
-      CLIENTCA="$PWD/$1"
+      CLIENTCA="$(realpath $1)"
       ;;
     "--jobs"|"-j")
       if echo "$2" | grep -qx "[1-9][0-9]*"; then
@@ -142,16 +142,10 @@ if [ "$USE_KOJI" ]; then
 
   if [ ! -f /etc/koji.conf ]; then
     echo "Setting up koji config . . ."
-    sudo cp projects/common/koji-client-files/koji.conf /etc
+    sudo cp projects/common/conf/koji.conf /etc
   fi
 fi
 
-if [ ! -f /etc/mock/clear.cfg ]; then
-  echo "Setting up mock config . . ."
-  sudo mkdir -p /etc/mock
-  sudo cp projects/common/koji-client-files/clear.cfg /etc/mock
-fi
-
 echo "Adding user to kvm group . . ."
 sudo usermod -a -G kvm $USER
 
@@ -163,6 +157,9 @@ if [ -n "$PACKAGE_REPOS" ]; then
   make ${JOBS_ARG} clone-packages
 fi
 
+echo "Creating mix workspace . . ."
+mkdir -p mix
+
 if [ "$USE_KOJI" ]; then
   echo "Testing koji installation . . ."
   if koji moshimoshi; then