Update packages file for version 23110

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile.common

@@ -20,8 +20,7 @@ RPMS ?= $(LATEST_RPMS)
 WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
 
 # MOCK_CONFIG_VAL is set in Makefile.shared
-MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL)
-
+MOCK ?= $(WITH_SUDO) /usr/bin/mock -r $(MOCK_CONFIG_VAL) $(MOCK_OPTS)
 
 ifneq ($(wildcard upstream),)
 __allsources := $(notdir $(strip $(shell cat upstream)))
@@ -89,6 +88,7 @@ prebuild-checks:
 #help files and rpms in the results directory of your package.  If there
 #help are mutlple tar.gz files for a package, make will do make sources,
 #help which creates a src.rpm file from the spec file.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options.
 build: prebuild-checks configemail $(SPECFILE) upstream $(SRPMFILE)
 	$(MOCK) $(SRPMFILE) --result=results/ --no-cleanup-after --uniqueext=$(PKG_NAME)
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
@@ -150,8 +150,10 @@ pullrebase:
 
 #help autospec: automatically generates a specfile. If there is
 #help already a specfile, it will be overwritten. Several files used by
-#help autospec will be created in the process. For more information, see
-#help the project at https://github.com/clearlinux/autospec
+#help autospec will be created in the process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: pullrebase localreponotice
 	rm -f results/*src.rpm
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
@@ -166,6 +168,7 @@ autospec: pullrebase localreponotice
 		--name $(PKG_NAME) \
 		--archives $(ARCHIVES) \
 		--mock-config $(MOCK_CONFIG_VAL) \
+		--mock-opts "$(MOCK_OPTS)" \
 		$${SETVERSION:+ --version $${SETVERSION}} \
 		${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 		$(firstword $(NEWURL) $(URL));
@@ -205,30 +208,12 @@ bumpnogit:
 
 $(__missingsources): sources
 
-#help sources: If SOURCES_URL is defined, download required upstream source
-#help files from that location. Otherwise, try to extract source files from the
-#help SRPM in the latest Clear Linux release, as specified by the LATEST_SRPMS
-#help variable. This will run automatically, as a dependency. NOTE: A Koji
-#help server can make use of this "make sources" command, since it lives in a
-#help repo named "common". If you use this makefile in Koji, ensure
-#help "make sources" continues to work for both remote and local builds, since
-#help it is a prerequisite of several commands for building packages.
-sources: upstream
-ifneq ($(strip $(SOURCES_URL)),)
-	while read u; do \
-		case "$$u" in \
-		    "")    continue ;; \
-		    *://*) n="$$u" ;; \
-		    *)     n="$(SOURCES_URL)/$$u" ;; \
-		esac; \
-		curl --fail --noproxy "*" "$$n" -o `basename "$$n"`; \
-	done < upstream
-else
-	@tmp=$$(mktemp -d -p "."); \
+# First argument is a Clear Linux build number (e.g. 22000)
+define fetch-from-srpm
+	tmp=$$(mktemp -d -p "."); \
 	srpm=$$(rpmspec --srpm -q --qf "%{NVR}.src.rpm\n" $(SPECFILE)); \
-	if cd "$$tmp" && ! curl -f -L -O $(LATEST_SRPMS)/$$srpm; then \
-		echo "Failed to download $$srpm from latest Clear Linux release." >&2; \
-		echo "Run \"git pull\" and try again." >&2; \
+	if cd "$$tmp" && ! curl -s -S -f -L -O $(MIRROR_CURL_OPTS) $(DOWNLOAD_MIRROR)/releases/$(1)/clear/source/SRPMS/$$srpm; then \
+		echo "Failed to download $$srpm from Clear Linux release $(1)." >&2; \
 		cd .. && rm -rf "$$tmp"; \
 		exit 1; \
 	fi; \
@@ -236,14 +221,58 @@ else
 	cd ..; \
 	srclist=$$(rpmspec --srpm -q --qf "[%{SOURCE}\n]" $(SPECFILE)); \
 	for s in $$srclist; do \
-		if ! mv "$$tmp"/$$s .; then \
+		if ! mv --no-clobber "$$tmp"/$$s .; then \
 			echo "Missing source file \"$$s\" in $${srpm}." >&2; \
 			rm -rf "$$tmp"; \
 			exit 1; \
 		fi; \
 		echo "Retrieved source file $$s ..."; \
 	done; \
-	rm -rf "$$tmp";
+	rm -rf "$$tmp"
+endef
+
+#help sources: If SOURCES_URL is defined, download required upstream source files
+#help from that location. Otherwise, try to download source files according to the
+#help URLs listed in the spec file. If any of the source files fail to download,
+#help check for the relevant SRPM from published releases of Clear Linux OS, and
+#help extract the files if found. Note that SRPMs are taken from the DOWNLOAD_MIRROR
+#help location. This will run automatically, as a dependency. NOTE: A Koji server can
+#help make use of this "make sources" command, since it lives in a repo named
+#help "common". If you use this makefile in Koji, ensure "make sources" continues to
+#help work for both remote and local builds, since it is a prerequisite of several
+#help commands for building packages.
+ifneq ($(strip $(SOURCES_URL)),)
+sources: upstream
+	while read u; do \
+		case "$$u" in \
+			"")    continue ;; \
+			*://*) n="$$u" ;; \
+			*)     n="$(SOURCES_URL)/$$u" ;; \
+		esac; \
+		if ! curl --fail -L -o `basename "$$n"` $(SOURCES_CURL_OPTS) "$$n"; then \
+			echo "Failed to download $$n"; \
+			exit 1; \
+		fi; \
+	done < upstream
+else
+sources:
+	@$(MAKE) generateupstream; \
+	[ $$? -eq 0 ] && exit 0; \
+	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
+	for b in $$latest_builds; do \
+		url="$(DOWNLOAD_MIRROR)/releases/$$b/clear/source/package-sources"; \
+		echo "Checking for source files in build $$b ..."; \
+		if grep -q "$$nvr" <(curl -s -f -L $(MIRROR_CURL_OPTS) $$url); then \
+			$(call fetch-from-srpm,$$b); \
+			if [ $$? -eq 0 ]; then \
+				echo "Source files retrieved from build $$b"; \
+				exit 0; \
+			fi; \
+		fi; \
+	done; \
+	echo "Source files not found for package"; \
+	exit 1
 endif
 
 prekoji-checks:
@@ -257,8 +286,10 @@ koji: prekoji-checks kojidef
 	@if ! git branch | grep -q -E '^\* master'; then \
 		echo "Error: Must be on the master branch to submit to koji" >&2; \
 		exit 1; \
+	fi; \
+	if git rev-parse --verify --quiet origin/master > /dev/null; then \
+		git pull --rebase; \
 	fi
-	git pull --rebase
 	git tag $(SRPMVERS)
 	git push origin master refs/tags/$(SRPMVERS)
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
@@ -362,18 +393,25 @@ install:
 
 #help generateupstream: Run this rule to create or update the 'upstream' file
 #help by downloading the upstream source tarballs listed in the spec file and
-#help calculating their hashes. Autospec performs this step automatically, so ignore
-#help it for packages managed with autospec.
+#help calculating their hashes. Autospec performs this step automatically, so
+#help ignore it for packages managed with autospec. Additional curl options
+#help may be used for downloading the upstream source tarballs by defining the
+#help CURL_OPTS variable for this rule.
 generateupstream:
-	@rm -f upstream
-	@urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | grep Source | cut -d: -f2- | grep '://'); \
+	@[ -e upstream ] && mv upstream upstream.bak; \
+	urls=$$(rpmspec -D '_vendor clr' -P $(SPECFILE) | sed -n "s|^Source[0-9]*[[:blank:]]*:[[:blank:]]*\(..*://..*\)$$|\1|p"); \
 	for url in $$urls; do \
-	    filename=$$(basename $$url); \
-	    if [ ! -e $$filename ]; then \
-		curl --fail -L -O $$url; \
-	    fi; \
-	    echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
+		filename=$$(basename $$url); \
+		if [ ! -e $$filename ]; then \
+			if ! curl --fail -L -O $(CURL_OPTS) $$url; then \
+				echo "Failed to download $$url"; \
+				[ -e upstream.bak ] && mv --no-clobber upstream.bak upstream; \
+				exit 1; \
+			fi; \
+		fi; \
+		echo $$(sha1sum $$filename | cut -d\  -f1)/$$filename >> upstream; \
 	done
+	@rm -f upstream.bak
 	@cat upstream
 
 #help drop-abandoned: Remove all unused patches from the git tree

Makefile.config

@@ -26,6 +26,9 @@ CGIT_BASE_URL =
 # Location of upstream tarball cache used by 'make build', 'make sources', etc.
 SOURCES_URL =
 
+# Additional curl options to use when downloading files from SOURCES_URL
+SOURCES_CURL_OPTS =
+
 # Location where package repos are hosted.
 PKG_BASE_URL = https://github.com/clearlinux-pkgs
 
@@ -38,6 +41,9 @@ PROJ_PKGS = autospec clr-bundles common
 # Clear Linux download mirror to use
 DOWNLOAD_MIRROR = https://cdn.download.clearlinux.org
 
+# Additional curl options to use when downloading files from DOWNLOAD_MIRROR
+MIRROR_CURL_OPTS =
+
 # Location of release content from latest Clear Linux release
 LATEST_RELEASE = $(DOWNLOAD_MIRROR)/current
 
@@ -48,17 +54,17 @@ LATEST_SRPMS = $(LATEST_RELEASE)/source/SRPMS
 PACKAGES_FILE = $(TOPLVL)/projects/common/packages
 
 # Configuration file for autospec.
-AUTOSPEC_CONF = $(TOPLVL)/projects/common/autospec.conf
+AUTOSPEC_CONF = $(TOPLVL)/projects/common/conf/autospec.conf
 
 # Set this variable to any non-empty value to use the package mappings stored
 # in "pkg-mapping" for cloning/pulling package repos.
 USE_PACKAGE_MAPPING = yes
 
-# Package manager config file. Default to the in-tree copy.
-PM_CONF = $(TOPLVL)/projects/common/image-creator/yum.conf
+# Package manager config file. Default to the in-tree copy of dnf.conf.
+PM_CONF = $(TOPLVL)/projects/common/conf/dnf.conf
 
 # Mock configuration file. Default to the in-tree copy.
-MOCK_CONF = $(TOPLVL)/projects/common/koji-client-files/clear.cfg
+MOCK_CONF = $(TOPLVL)/projects/common/conf/clear.cfg
 
 
 # Include any site local configuration variables. These variables may override

Makefile.shared

@@ -14,9 +14,9 @@ define loopup
 	@sudo partprobe /dev/loop$(DEVICE)
 	@sleep 1
 	@if [ -e /dev/loop$(DEVICE)p3 ]; then \
-	   sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p3 $(TOPLVL)/image; \
 	else \
-	   sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
+		sudo mount /dev/loop$(DEVICE)p2 $(TOPLVL)/image; \
 	fi
 	@sudo mount /dev/loop$(DEVICE)p1 $(TOPLVL)/image/boot
 endef

Makefile.toplevel

@@ -10,6 +10,7 @@ include $(TOPLVL)/projects/common/Makefile.shared
 SHELL   = /bin/bash
 
 PKGS    = $(shell cat $(PACKAGES_FILE))
+PKGS_LOCAL = $(sort $(filter ${PKGS},$(subst packages/,,$(wildcard packages/*))))
 
 proj_PKGS = $(addprefix proj_,$(PROJ_PKGS))
 clone_PKGS = $(addprefix clone_,$(PKGS))
@@ -64,7 +65,7 @@ cvecheck:
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
 .PHONY: pull ${PULL_PKGS} 
-PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,$(PKGS))
+PULL_PKGS:= $(addprefix PULL_projects/,$(PROJ_PKGS)) $(addprefix PULL_packages/,${PKGS_LOCAL})
 ${PULL_PKGS}:
 	@p=$(patsubst PULL_%,%,$@) ; \
 	if [ ! -d "$$p/.git" ]; then echo "Nothing to pull for $$p - $$p/.git missing"; exit 0; fi; \
@@ -191,12 +192,16 @@ provides:
 
 #help autospecnew: Creates a new autospec package with for a given URL=$(URL)
 #help with NAME=$(NAME). Several files used by autospec will be created in the
-#help process. For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
+#help process.
+#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
+#help to autospec.
+#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
 autospecnew: localreponotice
 	@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
+	-$(MAKE) clone_$(NAME)
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
@@ -215,6 +220,7 @@ autospecnew: localreponotice
 			--name $(NAME) \
 			--archives $(ARCHIVES) \
 			--mock-config $(MOCK_CONFIG_VAL) \
+			--mock-opts "$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
 			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 			$(URL); \

README.md

@@ -31,43 +31,13 @@ $ ./user-setup.sh
 After the script completes, make sure to logout and login again to complete the
 setup process.
 
-The script either accepts no options, or all (3) options in case you are
-configuring the Koji CLI for remote building on a Koji server. The options are
-documented in the script's `--help` output.
+The script accepts several options, or no options at all. The options are
+documented in the script's `--help` output. Note that if you are supplying any
+of the three Koji cert options (`-k`, `-s`, or `-c`), the other two options
+must be supplied as well.
 
-### Manual setup
-
-On your Clear Linux system, create a workspace for Clear Linux development
-work:
-
-```
-$ mkdir clearlinux
-```
-
-Clone this repo into a `projects` directory within the workspace:
-
-```
-$ cd clearlinux
-$ mkdir projects
-$ git clone https://github.com/clearlinux/common projects/common
-```
-
-Create the toplevel tooling Makefile:
-
-```
-$ ln -s projects/common/Makefile.toplevel Makefile
-```
-
-Clone all Clear Linux package and project repositories:
-
-```
-$ make clone
-```
-
-Note: You can clone the repos in parallel by using make's `-j` option.
-
-At this point, the `packages` directory will contain all Clear Linux package
-repos, and `projects` will contain common, clr-bundles, and autospec repos.
+If you do not wish to run the user-setup script, see the "Manual setup" section
+below for hints about how to initialize the tooling workspace.
 
 ## Example usage
 
@@ -141,3 +111,66 @@ package, a `make bump` command is available for this purpose.
 $ make bump
 $ make build
 ```
+
+## Other topics
+
+### Customizing the mock config
+
+In the past, the various make commands that call `mock` required a mock config
+installed at `/etc/mock/clear.cfg`. However, at present, the commands will
+instead use the mock config within this repo (`conf/clear.cfg`).
+
+If you wish to use a custom mock config, you must override the `MOCK_CONF`
+variable to specify a different value to pass to mock's `-r` option. The value
+is either a full path that ends with `.cfg`, or a config NAME installed at
+`/etc/mock/<NAME>.cfg`. You can override the `MOCK_CONF` config variable by
+redefining it in `Makefile.config.site_local`, which must reside at the
+toplevel directory in this repo.
+
+For example, to retain the old behavior of mock using `/etc/mock/clear.cfg`,
+add this line to `Makefile.config.site_local`:
+
+```
+MOCK_CONF = /etc/mock/clear.cfg
+```
+
+If `Makefile.config.site_local` doesn't exist already, create it.
+
+### Manual setup
+
+If you did not run the user-setup script (see "Automated setup" section above),
+you will want to set up the developer tooling workspace manually. This section
+provides general documentation for the manual setup process, and it is not
+meant to be exhaustive.
+
+On your Clear Linux system, create a workspace for Clear Linux development
+work:
+
+```
+$ mkdir clearlinux
+```
+
+Clone this repo into a `projects` directory within the workspace:
+
+```
+$ cd clearlinux
+$ mkdir projects
+$ git clone https://github.com/clearlinux/common projects/common
+```
+
+Create the toplevel tooling Makefile:
+
+```
+$ ln -s projects/common/Makefile.toplevel Makefile
+```
+
+Clone all Clear Linux package and project repositories:
+
+```
+$ make clone
+```
+
+Note: You can clone the repos in parallel by using make's `-j` option.
+
+At this point, the `packages` directory will contain all Clear Linux package
+repos, and `projects` will contain common, clr-bundles, and autospec repos.

conf/README.md

@@ -0,0 +1,31 @@
+# Configuration files
+
+This directory contains various configuration files used by the developer
+tooling framework.
+
+Unless otherwise noted in the config file documentation below, the config files
+can be modified with custom, site-local changes by modifying a variable in
+`Makefile.config.site_local` to point to a location of your choice (e.g.
+somewhere in `/etc`). The framework will then read the config file from the
+location you have specified instead, so copy the in-tree copy to that location
+as a starting point if you need to.
+
+* `autospec.conf`: default Autospec configuration file used by `make autospec`
+  and `make autospecnew`. All configuration is commented out by default, so
+  default Autospec settings are used. Config variable: `AUTOSPEC_CONF`
+
+* `clear.cfg`: default Mock configuration file used by various targets that
+  call out to Mock. Config variable: `MOCK_CONF`
+
+* `dnf.conf`: DNF conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `yum.conf`: YUM conf for use as the package manager configuration file, made
+  available for the framework's local repo support. Config variable: `PM_CONF`
+
+* `koji.conf`: template Koji configuration file. It is installed by
+  `user-setup.sh` to `/etc/koji.conf`, and the in-tree copy is not used. If you
+  are using Koji with the framework, you will need to modify `/etc/koji.conf`
+  according to how the Koji instance is set up for your environment. Configuring
+  the installation location via `Makefile.config.site_local` is not possible at
+  the moment but is a planned feature.

conf/autospec.conf

@@ -18,6 +18,6 @@
 # an absolute path or a path relative to this autospec.conf.
 #packages_file =
 
-# Path to yum config, used for generating whatrequires. It can be an absolute
+# Path to dnf/yum config, used for generating whatrequires. It can be an absolute
 # path or a path relative to this autospec.conf.
 #yum_conf =

conf/clear.cfg

@@ -10,13 +10,14 @@ config_opts['releasever'] = 'clear'
 config_opts['package_manager'] = 'dnf'
 
 
-config_opts['yum.conf'] = """
+config_opts['dnf.conf'] = """
 
 [main]
-cachedir=/var/cache/yum
+cachedir=/var/cache/dnf
+keepcache=1
 debuglevel=1
 reposdir=/dev/null
-logfile=/var/log/yum.log
+logfile=/var/log/dnf.log
 retries=20
 obsoletes=1
 gpgcheck=0

conf/dnf.conf

@@ -0,0 +1,24 @@
+[main]
+keepcache=1
+debuglevel=1
+logfile=/var/log/dnf.log
+exactarch=1
+obsoletes=1
+gpgcheck=0
+plugins=0
+retries=10
+installonly_limit=3
+reposdir=/root/mash
+
+[clear]
+name=Clear
+baseurl=https://cdn.download.clearlinux.org/current/x86_64/os/
+enabled=1
+gpgcheck=0
+
+[clear-source]
+name=Clear sources
+failovermethod=priority
+baseurl=https://cdn.download.clearlinux.org/current/source/SRPMS/
+enabled=1
+gpgcheck=0

packages

@@ -7,6 +7,8 @@ FreeRDP
 GConf
 Jinja2
 Keras
+Keras_Applications
+Keras_Preprocessing
 LS_COLORS
 LVM2
 LibRaw
@@ -25,6 +27,7 @@ OpenSP
 Paste
 PasteDeploy
 Pillow
+PyMySQL
 PyWavelets
 PyYAML
 Pygments
@@ -63,8 +66,10 @@ R-ICSNP
 R-IRdisplay
 R-IRkernel
 R-ISwR
+R-Iso
 R-Lahman
 R-LearnBayes
+R-MCMCpack
 R-Matching
 R-MatrixModels
 R-ModelMetrics
@@ -129,6 +134,7 @@ R-bayesplot
 R-bdsmatrix
 R-betareg
 R-bibtex
+R-biglm
 R-bindr
 R-bindrcpp
 R-bipartite
@@ -202,6 +208,7 @@ R-e1071
 R-earth
 R-ellipse
 R-energy
+R-estimability
 R-evaluate
 R-expint
 R-expm
@@ -218,14 +225,17 @@ R-flexmix
 R-forcats
 R-foreach
 R-formatR
+R-fossil
 R-fpc
 R-futile.logger
 R-futile.options
+R-future
 R-gam
 R-gamm4
 R-gbm
 R-gclus
 R-gdata
+R-geepack
 R-geometry
 R-getopt
 R-ggjoy
@@ -234,6 +244,7 @@ R-ggplot2movies
 R-ggridges
 R-git2r
 R-glmnet
+R-globals
 R-glue
 R-gmodels
 R-gmp
@@ -253,8 +264,10 @@ R-htmltools
 R-htmlwidgets
 R-httpuv
 R-httr
+R-hunspell
 R-ica
 R-igraph
+R-import
 R-inline
 R-inum
 R-ipred
@@ -270,23 +283,28 @@ R-labelled
 R-laeken
 R-lambda.r
 R-lars
+R-later
 R-latticeExtra
 R-lava
 R-lazyeval
 R-lda
+R-leaflet
 R-leaps
 R-lfe
 R-libcoin
+R-listenv
 R-lle
 R-lme4
 R-lmtest
 R-locfit
 R-loo
 R-lpSolve
+R-lsmeans
 R-lubridate
 R-magic
 R-magrittr
 R-manipulate
+R-manipulateWidget
 R-mapproj
 R-maps
 R-maptools
@@ -295,6 +313,7 @@ R-matrixStats
 R-matrixcalc
 R-maxLik
 R-mclust
+R-mcmc
 R-mda
 R-memisc
 R-memoise
@@ -313,6 +332,7 @@ R-modelr
 R-modeltools
 R-mondate
 R-multcomp
+R-multcompView
 R-multiwayvcov
 R-munsell
 R-mvoutlier
@@ -358,9 +378,11 @@ R-pracma
 R-praise
 R-prettyunits
 R-princurve
+R-processx
 R-prodlim
 R-profileModel
 R-progress
+R-promises
 R-proto
 R-proxy
 R-pryr
@@ -419,6 +441,7 @@ R-setRNG
 R-sfsmisc
 R-sgeostat
 R-shape
+R-shapefiles
 R-shiny
 R-shinyjs
 R-shinystan
@@ -439,6 +462,8 @@ R-spatstat
 R-spatstat.data
 R-spatstat.utils
 R-spdep
+R-speedglm
+R-spelling
 R-spls
 R-statmod
 R-statnet.common
@@ -461,6 +486,7 @@ R-tidyr
 R-tidyselect
 R-tidyverse
 R-timeDate
+R-tinytex
 R-tis
 R-tm
 R-topicmodels
@@ -477,6 +503,7 @@ R-vegan
 R-viridis
 R-viridisLite
 R-visNetwork
+R-webshot
 R-whisker
 R-withr
 R-xml2
@@ -484,6 +511,7 @@ R-xtable
 R-xts
 R-yaml
 R-zCompositions
+R-zip
 R-zoo
 Remmina
 SDL
@@ -580,6 +608,7 @@ backports.weakref
 baobab
 bash
 bash-completion
+bash.static
 bash_kernel
 bashdb
 bats
@@ -589,7 +618,6 @@ bcrypt
 bdftopcf
 beautifulsoup4
 beignet
-bigreqsproto
 bind-utils
 binutils
 bison
@@ -597,6 +625,7 @@ bkcharts
 bleach
 blktrace
 bluez
+bmap-tools
 bndl-lamp-basic
 bokeh
 boost
@@ -637,6 +666,8 @@ cgit
 chardet
 check
 cheese
+cheroot
+cherrypy
 chrome-gnome-shell
 chrony
 chrpath
@@ -683,9 +714,11 @@ colord
 colord-gtk
 comedilib
 compat-SDL2-soname1
+compat-cryptsetup-soname4
 compat-efivar-soname0
 compat-fuse-soname2
 compat-gegl
+compat-gegl-0.3
 compat-glew-soname1
 compat-gtksourceview-soname3
 compat-guile-soname20
@@ -701,14 +734,15 @@ compat-opencv-soname33
 compat-protobuf-soname14
 compat-readline
 component
-compositeproto
 compute-image-packages
 configobj
 configparser
+confuse
 connect-proxy
 connman
 connman-gtk
 conntrack-tools
+console-autostart
 containerd
 contextlib2
 coreutils
@@ -735,8 +769,8 @@ cups-filters
 cups-pk-helper
 curl
 cve-check-tool
+cxxfilt
 cycler
-damageproto
 dapl
 darktable
 dask
@@ -744,20 +778,23 @@ db
 dbus
 dbus-glib
 dconf
+dconf-editor
 deap
 decorator
 defusedxml
 dejagnu
 desktop-file-utils
 dfc
+dfu-util
 dhcp
 dialog
 diffoscope
 diffstat
 diffutils
 dist-pam-configs
+distro
+dmenu
 dmidecode
-dmxproto
 dnf
 dnf-plugins-core
 dnsmasq
@@ -775,9 +812,9 @@ double-conversion
 doxygen
 dpdk
 dracut
-dri2proto
-dri3proto
 dstat
+dtc
+dyskctl
 e2fsprogs
 ebtables
 ecdsa
@@ -823,7 +860,6 @@ findutils
 fio
 firefox
 fish
-fixesproto
 fixtures
 flac
 flake8
@@ -839,7 +875,6 @@ font-adobe-75dpi
 font-bitstream-type1
 font-util
 fontconfig
-fontsproto
 freeglut
 freeipmi
 freetype
@@ -849,6 +884,8 @@ funcsigs
 functools32
 fuse
 futures
+fwupd
+fwupdate
 garcon
 gast
 gawk
@@ -874,6 +911,7 @@ gevent
 gexiv2
 gfbgraph
 gflags
+gftp
 ghostscript
 gimp
 git
@@ -885,9 +923,9 @@ glib
 glib-networking
 glibc
 glibmm
+glmark2
 global
 glog
-glproto
 glslang
 glu
 glusterfs
@@ -1009,6 +1047,10 @@ hyperscan
 hyperstart
 hypothesis
 i2c-tools
+i3
+i3lock
+i3status
+iasimage
 ibus
 icalendar
 icdiff
@@ -1020,10 +1062,12 @@ imagesize
 imapfilter
 indent
 infiniband-diags
+influxdb
 iniparse
 init-rdahead
 inotify-tools
-inputproto
+intel-gpu-tools
+intel-hybrid-driver
 intltool
 invm-frameworks
 ioping
@@ -1076,8 +1120,13 @@ jupyter_core
 jupyterlab
 jupyterlab_launcher
 kafka-dep
+kata-image
+kata-proxy
+kata-qemu-lite
+kata-runtime
+kata-shim
 kbd
-kbproto
+kernel-install
 kexec-tools
 keyring
 keyutils
@@ -1148,6 +1197,7 @@ libdmx
 libdnf
 libdrm
 libepoxy
+libev
 libevdev
 libevent
 libexif
@@ -1160,6 +1210,7 @@ libgee
 libgepub
 libgfortran-avx
 libgfortran-compat-soname3
+libgfortran-compat-soname4
 libgit2
 libglade
 libgnome-keyring
@@ -1179,10 +1230,12 @@ libisofs
 libjpeg-turbo
 libjpeg-turbo-soname8
 libksba
+libmbim
 libmediaart
 libmnl
 libmspack
 libmtp
+libmypaint
 libndp
 libnetfilter_conntrack
 libnetfilter_cthelper
@@ -1203,6 +1256,7 @@ libpipeline
 libpng
 libpthread-stubs
 libpwquality
+libqmi
 librepo
 librsvg
 librtlsdr
@@ -1211,6 +1265,8 @@ libsass
 libseccomp
 libsecret
 libsigc++
+libsmbios
+libsmi
 libsndfile
 libsolv
 libsoup
@@ -1229,6 +1285,7 @@ libunistring
 libunwind
 libusb
 libuser
+libuv
 libva
 libva-intel-driver
 libva-utils
@@ -1252,7 +1309,9 @@ libxml2-legacy
 libxshmfence
 libxslt
 libyami
+libyami-utils
 libzip
+libzmq
 linecache2
 links
 linux
@@ -1263,12 +1322,15 @@ linux-gce
 linux-hyperv
 linux-hyperv-lts
 linux-hyperv-mini
+linux-kata
 linux-kvm
 linux-libc-headers
 linux-lts
+linux-oracle
 linux-pk414
 linux-steam-integration
 linux-tools
+linuxptp
 llvm
 llvmlite
 lmdb
@@ -1297,13 +1359,17 @@ mc
 mccabe
 mcelog
 mdadm
+meld
+memkind
 memshrinker
 mesa
 mesa-demos
 meson
 metacity
+metrics-discovery
 micro-config-drive
 micro-config-drive-aws
+micro-config-drive-oci
 mido
 minetest
 minetest_game
@@ -1316,6 +1382,8 @@ mkosi
 mock
 mod_wsgi
 moka-icon-theme
+more-itertools
+moreutils
 mosh
 motd-update
 motif
@@ -1325,6 +1393,7 @@ mozjs52
 mpc
 mpfr
 mpg123
+mpi4py
 msgpack-python
 msmtp
 msr-tools
@@ -1336,6 +1405,7 @@ murrine
 musl
 mutt
 mutter
+mypaint-brushes
 mypy
 nagios-plugins
 nano
@@ -1368,6 +1438,7 @@ ninja
 nlopt
 nodejs
 nose
+nose-cov
 nose-parameterized
 nosync
 notebook
@@ -1381,12 +1452,14 @@ nss
 nss-altfiles
 ntfs-3g
 numactl
+numatop
 numba
 numpy
 numpydoc
 nvme-cli
-nvml
+oath-toolkit
 oauth2client
+ocaml
 ocl-icd
 offlineimap
 olefile
@@ -1415,6 +1488,7 @@ opusfile
 orc
 orca
 ordereddict
+osinfo-db-tools
 ostree
 ovirt-guest-agent
 p11-kit
@@ -1437,6 +1511,7 @@ patch
 patchutils
 pavucontrol
 pbr
+pbr-legacy
 pbzip2
 pciutils
 pcre
@@ -1472,6 +1547,7 @@ perl-ExtUtils-Helpers
 perl-ExtUtils-InstallPaths
 perl-File-Copy-Recursive
 perl-File-Listing
+perl-File-MMagic
 perl-File-ShareDir
 perl-File-ShareDir-Install
 perl-File-Slurp
@@ -1566,20 +1642,23 @@ pidgin-sipe
 pigz
 pinentry
 pip
+pip-legacy
 pixman
 pkcs11-helper
 pkg-config
 pluggy
 ply
+pmdk
 polkit
 poppler
+poppler-data
 popt
 portaudio
+portend
 posix_ipc
 postgresql
 powerline-fonts
 powertop
-presentproto
 pretend
 prettytable
 proc-macro2
@@ -1605,7 +1684,6 @@ pyatspi
 pycairo
 pycodestyle
 pycparser
-pycrypto
 pycurl
 pydot
 pydot_ng
@@ -1631,7 +1709,7 @@ python-augeas
 python-dateutil
 python-future
 python-gflags
-python-keyczar
+python-kconfiglib
 python-krbV
 python-magic
 python-memcached
@@ -1641,6 +1719,7 @@ python-multilib
 python-pkgconfig
 python-pocketlint
 python-polib
+python-rpm-legacypython
 python-subunit
 python-systemd
 python-tappy
@@ -1687,17 +1766,14 @@ qtxmlpatterns
 quagga
 quilt
 ragel
-randrproto
 rdma-core
 re2c
 readline
-recordproto
 redis
 redis-native
 redsocks
-renderproto
 requests
-resourceproto
+requests-mock
 rest
 retry_decorator
 retype
@@ -1708,6 +1784,7 @@ rpm
 rr
 rsa
 rsync
+rt-tests
 ruby
 runc
 rust-std32
@@ -1716,6 +1793,7 @@ rxvt-unicode
 s2tc
 s3transfer
 samba
+sane-backends
 sassc
 satyr
 sbc
@@ -1729,20 +1807,24 @@ scipy
 scons
 screen
 scripttest
-scrnsaverproto
 seaborn
 seahorse
 secretstorage
 sed
+serf
 setproctitle
 setserial
 setuptools
+setuptools-legacy
 setuptools_scm
+setuptools_scm_git_archive
 shadow
 shared-mime-info
 sharutils
 shim
 siege
+simg-tools
+simple-scan
 simplegeneric
 simplejson
 sip
@@ -1758,6 +1840,7 @@ soundmodem
 source-code-pro
 source-highlight
 sox
+spandsp
 spark-dep
 speex
 speexdsp
@@ -1776,7 +1859,9 @@ stack
 startup-notification
 stow
 strace
+stress-ng
 subunit
+subversion
 sudo
 sure
 suricata
@@ -1800,6 +1885,7 @@ tcpdump
 telemetrics-client
 telepathy-glib
 telepathy-logger
+tempora
 tensorboard
 tensorflow
 termcolor
@@ -1811,6 +1897,7 @@ testresources
 testscenarios
 testtools
 texinfo
+texlive
 tflearn
 thermal_daemon
 thin-provisioning-tools
@@ -1843,8 +1930,10 @@ tzdata
 tzlocal
 udisks2
 uhttpmock
+unbundle
 unicode-xid
 unifdef
+unison
 unittest2
 unzip
 upower
@@ -1863,7 +1952,6 @@ v4l-utils
 vala
 valgrind
 vcversioner
-videoproto
 vim
 vim-go
 vinagre
@@ -1873,10 +1961,10 @@ virt-viewer
 virtualenv
 vlc
 volume_key
-vorbis-tools
 vsqlite
 vte
 vulkan-sdk
+waffle
 waitress
 wayland
 wayland-protocols
@@ -1893,6 +1981,7 @@ wheel
 which
 widgetsnbextension
 wireless-regdb-master
+wireshark
 woff2
 wol
 wpa_supplicant
@@ -1905,11 +1994,13 @@ xauth
 xbitmaps
 xcb-proto
 xcb-util
+xcb-util-cursor
 xcb-util-image
 xcb-util-keysyms
+xcb-util-renderutil
 xcb-util-wm
+xcb-util-xrm
 xclip
-xcmiscproto
 xcursor-themes
 xcursorgen
 xdg-desktop-portal
@@ -1917,7 +2008,6 @@ xdg-user-dirs
 xdg-user-dirs-gtk
 xdg-utils
 xemacs
-xextproto
 xf86-input-evdev
 xf86-input-libinput
 xf86-input-mouse
@@ -1928,10 +2018,6 @@ xf86-video-ati
 xf86-video-fbdev
 xf86-video-nouveau
 xf86-video-vesa
-xf86bigfontproto
-xf86dgaproto
-xf86driproto
-xf86vidmodeproto
 xfburn
 xfce4-appfinder
 xfce4-dev-tools
@@ -1947,7 +2033,6 @@ xfdesktop
 xfsprogs
 xfwm4
 xhost
-xineramaproto
 xinit
 xkbcomp
 xkeyboard-config
@@ -1956,8 +2041,8 @@ xmlsec1
 xmlstarlet
 xmlto
 xorg-server
+xorgproto
 xorriso
-xproto
 xrandr
 xrdb
 xrdp

provides.sh

@@ -23,7 +23,7 @@ if [ "$#" -eq 0 ]; then
 fi
 
 if [ ! -e /etc/yum.conf ]; then
-  echo "Error: yum.conf is missing. Please copy projects/common/image-creator/yum.conf to /etc"
+  echo "Error: yum.conf is missing. Please copy projects/common/conf/yum.conf to /etc"
   exit 1
 fi
 

user-setup.sh

@@ -1,102 +1,131 @@
-#!/bin/sh
+#!/bin/bash
 
 SCRIPT=$(/usr/bin/basename $0)
 PEM=""
 SERVERCA=""
 CLIENTCA=""
+WORKSPACE="clearlinux"
+PACKAGE_REPOS=
 
 help() {
-	printf "%s\n" >&2 "Usage: $SCRIPT [options]" \
-			  "" \
-			  "Options:" \
-			  "-k --client-cert PEM_FILE: Enable client user cert for koji configuration; requires a PEM file argument" \
-			  "-s --server-ca PEM_FILE: Enable server CA cert for koji configuration; requires a PEM file argument" \
-			  "-c --client-ca PEM_FILE: Enable client CA cert for koji configuration; requires a PEM file argument" \
-			  ""
+  printf "%s\n" >&2 "Usage: $SCRIPT [options]" \
+    "" \
+    "Options:" \
+    "" \
+    "-d --directory NAME: Set up workspace in the given directory." \
+    "-a --clone-packages: Clone all package repos." \
+    "-j --jobs [NUM]: Clone repos with NUM jobs. If NUM is not given, it is set to the available CPU count." \
+    "" \
+    "-k --client-cert PEM_FILE: Enable client user cert for koji configuration; requires a PEM file argument" \
+    "-s --server-ca PEM_FILE: Enable server CA cert for koji configuration; requires a PEM file argument" \
+    "-c --client-ca PEM_FILE: Enable client CA cert for koji configuration; requires a PEM file argument" \
+    ""
+}
+
+error() {
+  echo -e "Error: $1\n" >&2
+  help
+  exit 1
 }
 
 while [ $# -gt 0 ]; do
-	case "$1" in
-		"--help"|"-h")
-			help
-			exit 0
-			;;
-		"--client-cert"|"-k")
-			shift
-			PEM="$PWD/$1"
-			;;
-		"--server-ca"|"-s")
-			shift
-			SERVERCA="$PWD/$1"
-			;;
-		"--client-ca"|"-c")
-			shift
-			CLIENTCA="$PWD/$1"
-			;;
-		*)
-			help
-			exit 1
-			;;
-	esac
-	shift
+  case "$1" in
+    "--help"|"-h")
+      help
+      exit 0
+      ;;
+    "--client-cert"|"-k")
+      shift
+      PEM="$(realpath $1)"
+      ;;
+    "--server-ca"|"-s")
+      shift
+      SERVERCA="$(realpath $1)"
+      ;;
+    "--client-ca"|"-c")
+      shift
+      CLIENTCA="$(realpath $1)"
+      ;;
+    "--jobs"|"-j")
+      if echo "$2" | grep -qx "[1-9][0-9]*"; then
+        shift
+        JOBS="$1"
+      elif [ -f /proc/cpuinfo ]; then
+        JOBS=$(grep -Ec '^processor.*:.*[0-9]+$' /proc/cpuinfo)
+      fi
+      ;;
+    "--directory"|"-d")
+      [ -z "$2" ] && error "Must supply a directory name to the -d option"
+      [ "${2:0:1}" = "-" ] && error "Directory name cannot begin with \"-\""
+      shift
+      WORKSPACE="$1"
+      ;;
+    "--clone-packages"|"-a")
+      PACKAGE_REPOS=1
+      ;;
+    *)
+      help
+      exit 1
+      ;;
+  esac
+  shift
 done
 
-error() {
-	echo -e "Error: $1\n" >&2
-	help
-	exit 1
-}
 
 if [ -z "$PEM" ] && [ -z "$SERVERCA" ] && [ -z "$CLIENTCA" ]; then
-	USE_KOJI=
+  USE_KOJI=
 else
-	if [ -z "$PEM" ] || [ -z "$SERVERCA" ] || [ -z "$CLIENTCA" ]; then
-		error "Must specify all three command line options (or none)"
-	fi
-	if [ ! -f "$PEM" ]; then
-		error "Missing koji client PEM key file"
-	fi
-	if [ ! -f "$SERVERCA" ]; then
-		error "Missing koji server CA PEM file"
-	fi
-	if [ ! -f "$CLIENTCA" ]; then
-		error "Missing koji client CA PEM file"
-	fi
-	USE_KOJI="yes"
+  if [ -z "$PEM" ] || [ -z "$SERVERCA" ] || [ -z "$CLIENTCA" ]; then
+    error "Must specify all three command line options (or none)"
+  fi
+  if [ ! -f "$PEM" ]; then
+    error "Missing koji client PEM key file"
+  fi
+  if [ ! -f "$SERVERCA" ]; then
+    error "Missing koji server CA PEM file"
+  fi
+  if [ ! -f "$CLIENTCA" ]; then
+    error "Missing koji client CA PEM file"
+  fi
+  USE_KOJI="yes"
+fi
+
+if [ -n "$JOBS" ]; then
+  JOBS_ARG="-j $JOBS"
+fi
+
+if [ -d "$WORKSPACE" ]; then
+  error "Directory \"$WORKSPACE\" already exists. \
+Either remove this workspace, or use a different workspace name."
 fi
 
 required_progs() {
-	local bindir="/usr/bin"
-	for f in git mock rpm rpmbuild ; do
-		[ ! -x "${bindir}/${f}" ] && missing+="${f} "
-	done
-	[ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
-	if [ -n "$missing" ]; then
-		echo "Install the following programs and re-run this script:" >&2
-		echo $missing >&2
-		echo 'All programs should be provided in the "os-clr-on-clr" bundle.' >&2
-		exit 1
-	fi
+  local bindir="/usr/bin"
+  for f in git mock rpm rpmbuild ; do
+    [ ! -x "${bindir}/${f}" ] && missing+="${f} "
+  done
+  [ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "
+  if [ -n "$missing" ]; then
+    echo "Install the following programs and re-run this script:" >&2
+    echo $missing >&2
+    echo 'All programs should be provided in the "os-clr-on-clr" bundle.' >&2
+    exit 1
+  fi
 }
 
 required_progs
 
-echo 'Initializing development workspace in "clearlinux" . . .'
-if [ -d "clearlinux" ]; then
-	echo 'Directory "clearlinux" already exists in current directory.' >&2
-	echo "Cannot initialize workspace." >&2
-	exit 1
-fi
+echo "Initializing development workspace in \"$WORKSPACE\" . . ."
 
-mkdir clearlinux
-cd clearlinux
+mkdir "$WORKSPACE"
+cd "$WORKSPACE"
 
 echo "Setting up common repo . . ."
 mkdir projects
 git clone https://github.com/clearlinux/common projects/common
 if [ $? -ne 0 ]; then
-	echo "Failed to clone common repo." >&2
-	exit 1
+  echo "Failed to clone common repo." >&2
+  exit 1
 fi
 
 # Finish setup for packages/projects hierarchy
@@ -105,48 +134,52 @@ mkdir -p packages/common
 ln -sf ../../projects/common/Makefile.common packages/common/Makefile.common
 
 if [ "$USE_KOJI" ]; then
-	echo "Setting up koji certs . . ."
-	mkdir -p ~/.koji
-	cp "$PEM" ~/.koji/client.crt
-	cp "$CLIENTCA" ~/.koji/clientca.crt
-	cp "$SERVERCA" ~/.koji/serverca.crt
+  echo "Setting up koji certs . . ."
+  mkdir -p ~/.koji
+  cp "$PEM" ~/.koji/client.crt
+  cp "$CLIENTCA" ~/.koji/clientca.crt
+  cp "$SERVERCA" ~/.koji/serverca.crt
 
-	if [ ! -f /etc/koji.conf ]; then
-		echo "Setting up koji config . . ."
-		sudo cp projects/common/koji-client-files/koji.conf /etc
-	fi
-fi
-
-if [ ! -f /etc/mock/clear.cfg ]; then
-	echo "Setting up mock config . . ."
-	sudo mkdir -p /etc/mock
-	sudo cp projects/common/koji-client-files/clear.cfg /etc/mock
+  if [ ! -f /etc/koji.conf ]; then
+    echo "Setting up koji config . . ."
+    sudo cp projects/common/conf/koji.conf /etc
+  fi
 fi
 
 echo "Adding user to kvm group . . ."
 sudo usermod -a -G kvm $USER
 
 echo "Cloning special project repositories . . ."
-make clone-projects
+make ${JOBS_ARG} clone-projects
 
-if [ -z "$NO_PACKAGE_REPOS" ]; then
-	echo "Cloning all package repositories . . ."
-	make clone-packages
+if [ -n "$PACKAGE_REPOS" ]; then
+  echo "Cloning all package repositories . . ."
+  make ${JOBS_ARG} clone-packages
 fi
 
+echo "Creating mix workspace . . ."
+mkdir -p mix
+
 if [ "$USE_KOJI" ]; then
-	echo "Testing koji installation . . ."
-	if koji moshimoshi; then
-		echo -en "\n************************\n\n"
-		echo "Koji installed and configured successfully"
-	else
-		echo -en "\n************************\n\n"
-		echo "Error with koji installation or configuration" >&2
-		exit 1
-	fi
+  echo "Testing koji installation . . ."
+  if koji moshimoshi; then
+    echo -en "\n************************\n\n"
+    echo "Koji installed and configured successfully"
+  else
+    echo -en "\n************************\n\n"
+    echo "Error with koji installation or configuration" >&2
+    exit 1
+  fi
 fi
 
 echo -en "\n************************\n"
 
-echo 'Workspace has been set up in the "clearlinux" directory'
+echo "Workspace has been set up in \"$WORKSPACE\""
+if [ -z "$PACKAGE_REPOS" ]; then
+  echo "NOTE: To clone all package repos, run \"cd $WORKSPACE; make [-j NUM] clone-packages\""
+  echo "NOTE: To clone a single package repo with NAME, run \"cd $WORKSPACE; make clone_NAME\""
+fi
 echo 'NOTE: logout and log back in to finalize the setup process'
+
+
+# vi: ft=sh sw=2 et sts=2