Update packages file for version 33100

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile.common

@@ -49,13 +49,6 @@ spdxcheck: $(SPECFILE)
 		if ! grep -qx "$$LICENSE" $(TOPLVL)/projects/common/licenses-{spdx,exceptions,extra}; then FAIL=yes; echo "ERROR: License '$$LICENSE' is not an allowed SPDX license ID"; fi; \
 	done; test -z "$$FAIL"
 
-#help cvecheck: Checks
-#help for common vulnerabilities and exposures in your package.
-cvecheck:
-	# https://github.com/ikeydoherty/cve-check-tool
-	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
-	cve-check-tool -M $(TOPLVL)/projects/common/mapping $(SPECFILE)
-
 #help clean: Cleans up the src.rpm file. but not the rpm files or log files.
 clean:
 	rm -f $(SRPMFILE)
@@ -179,9 +172,11 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		--name $(PKG_NAME) \
 		--archives $(ARCHIVES) \
 		--mock-config $(MOCK_CONFIG_VAL) \
-		--mock-opts "$(MOCK_OPTS)" \
+		--mock-opts="$(MOCK_OPTS)" \
 		$${SETVERSION:+ --version $${SETVERSION}} \
-		${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
+		$${NON_INTERACTIVE:+ --non_interactive} \
+		$${SKIP_GIT:+ --skip-git} \
+		$${CLEANUP:+ -C} \
 		$(firstword $(NEWURL) $(URL));
 	$(MAKE) link-new-rpms PKG_REPO_DIR="."
 	@$(MAKE) spdxcheck
@@ -206,12 +201,12 @@ for-review.txt:
 
 #help autospecnogit: Runs autospec, but does not create a commit
 autospecnogit:
-	$(MAKE) SKIP_GIT=--skip-git autospec
+	$(MAKE) autospec SKIP_GIT=1
 
 #help autospecnostate: Runs autospec, but cleans up mock chroots
 #help and disables interactive mode.
 autospecnostate:
-	$(MAKE) CLEANUP=-C NON_INTERACTIVE=--non_interactive autospec
+	$(MAKE) autospec CLEANUP=1 NON_INTERACTIVE=1
 
 scanlicense:
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --config $(AUTOSPEC_CONF) --license-only $(firstword $(NEWURL) $(URL)) --name $(PKG_NAME)
@@ -220,7 +215,7 @@ scanlicense:
 bump:
 	git pull --rebase
 	$(MAKE) bumpnogit
-	git add *.spec release
+	git add $(SPECFILE) release
 	git commit -a -m "version bump from $(shell rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)) to `rpmspec -D '_vendor clr' --srpm -q --queryformat '%{VERSION}-%{RELEASE}\n' $(SPECFILE)`"
 
 #help bumpnogit: Increments the release file by one and does not commit changes.
@@ -309,16 +304,21 @@ prekoji-checks:
 koji: prekoji-checks kojidef
 	@$(MAKE) spdxcheck
 	@$(MAKE) checkblacklist
-	@if ! git branch | grep -q -E '^\* master'; then \
+	@if [ "$$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then \
 		echo "Error: Must be on the master branch to submit to koji" >&2; \
 		exit 1; \
 	fi; \
-	if ! git diff --quiet HEAD ${SPECFILE}; then \
+	if ! git diff --quiet HEAD -- ${SPECFILE}; then \
 		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
 		exit 1; \
 	fi; \
-	if git rev-parse --verify --quiet origin/master > /dev/null; then \
+	git fetch; \
+	if git rev-parse --verify --quiet origin/master >/dev/null; then \
 		git pull --rebase; \
+	fi; \
+	if git ls-remote --tags --exit-code origin $(SRPMVERS) >/dev/null; then \
+		echo "Error: remote tag $(SRPMVERS) already exists" >&2; \
+		exit 1; \
 	fi
 	git tag $(SRPMVERS)
 	git push origin master refs/tags/$(SRPMVERS)

Makefile.shared

@@ -62,16 +62,48 @@ define remotepkgname
 $(if $(USE_PACKAGE_MAPPING),$$(awk -v P="$(1)" '$$1 == P { res=$$2 } END { print res ? res : P }' $(TOPLVL)/projects/common/pkg-mapping),$(1))
 endef
 
+# Clone repo from the configured gitolite instance, but only if the repo exists
+# on that instance and matches the pattern "packages/..*".
+define try-clone-gitolite
+rpkg=$(call remotepkgname,$(1)); \
+if ssh $(GITOLITE_BASE_URL) info "packages/$$rpkg" | grep --quiet "packages/$$rpkg$$"; then \
+	git clone $(PKG_BASE_URL)/$$rpkg packages/$(1); \
+	cd packages/$(1); \
+	$(call gitoliteurl,packages/$(1)); \
+	$(call subjectprefix,$(1)); \
+fi
+endef
+
+# Clone repo from the configured package repo hosting location, as defined by
+# the PKG_BASE_URL variable, but only if the repo exists.
+define try-clone-other
+rpkg=$(call remotepkgname,$(1)); \
+if git ls-remote $(PKG_BASE_URL)/$$rpkg &> /dev/null; then \
+	git clone $(PKG_BASE_URL)/$$rpkg packages/$(1); \
+	cd packages/$(1); \
+	$(call subjectprefix,$(1)); \
+fi
+endef
+
+# First, try running `make clone_PKG`, where PKG is the first argument. If that
+# fails, proceed with fallback cloning strategies.
+define clone-if-available
+$(MAKE) clone_$(1) || { \
+	$(if $(GITOLITE_BASE_URL),$(call try-clone-gitolite,$(1)),$(call try-clone-other,$(1))); \
+}
+endef
+
+
 # Enables the local RPM repo by installing package manager and Mock configs.
 # First argument is the path to the package manager config, and the second
 # argument is the path to the mock config. Each of these config files is
 # augmented for local repo support.
 define localrepoenable
 	@cp $(1) $(TOPLVL)/repo/yum.conf
-	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/yum.conf
+	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $(TOPLVL)/repo/yum.conf
 	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/yum.conf
 	@cp $(2) $(TOPLVL)/repo/clear.cfg
-	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\n" >> $(TOPLVL)/repo/clear.cfg
+	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $(TOPLVL)/repo/clear.cfg
 	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/clear.cfg
 	@sed -i '/^"""/d; $$a"""' $(TOPLVL)/repo/clear.cfg
 endef

Makefile.toplevel

@@ -34,7 +34,7 @@ $(proj_PKGS):
 	git clone $(PRJ_BASE_URL)/$$proj projects/$$proj; \
 	cd projects/$$proj; \
 	if [ "$$proj" = "autospec" ] ; then \
-		git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+		git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 	else \
 		$(call gitoliteurl,projects/$$proj); \
 	fi; \
@@ -56,12 +56,6 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	$(call subjectprefix,$$pkg); \
 	)
 
-#help cvecheck: Checks for common vulnerabilities and exposures in your code.
-cvecheck:
-	# https://github.com/ikeydoherty/cve-check-tool
-	# Installation: https://github.com/ikeydoherty/cve-check-tool/wiki
-	cve-check-tool -n -M $(TOPLVL)/projects/common/mapping $(PACKAGES_FILE)
-
 #help pull: Performs a git pull --rebase for each package repo, avoiding the creation
 #help of merge commits, while displaying any changes since your last pull. It is
 #help silent if there are no changes.
@@ -83,7 +77,7 @@ ${PULL_PKGS}:
 			git --no-pager log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit $$O..$$N; echo; \
 		fi ; \
 		if [ "$$p" = "projects/autospec" ] ; then \
-			git remote set-url --push origin https://github.com/clearlinux/autospec.git; \
+			git remote set-url --push origin git@github.com:clearlinux/autospec.git; \
 		else \
 			$(call gitoliteurl,$$p); \
 		fi ; \
@@ -172,6 +166,15 @@ diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 		(cd $$p ; git status | grep -q 'nothing to commit, working [^ ]* clean' || (echo "Uncommitted changes in: $$p"; git diff |diffstat -p1)) ;\
 	done
 
+#help log: Run git log origin/master..HEAD for all package repos, thus
+#help displaying any committed but unpushed changes.
+log: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
+	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
+		( cd $$p; if [ "$$(git rev-parse --verify HEAD)" != "$$(git rev-parse --verify origin/master)" ] ; then \
+		echo $$p ; git --no-pager log --pretty="format:%s" origin/master..HEAD ; echo ; echo;\
+		fi ; ) \
+	done
+
 #help versions: Displays the version of each package in the ./packages tree.
 versions: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in $(addprefix packages/,$(PKGS_LOCAL)); do \
@@ -202,7 +205,7 @@ autospecnew: preautospecnew-checks localreponotice
 		echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
 		exit 1; \
 	fi
-	-$(MAKE) clone_$(NAME)
+	$(call clone-if-available,$(NAME))
 	@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
 		echo "no remote repository found, creating new package repository and running autospec"; \
 		mkdir -p $(TOPLVL)/packages/$(NAME); \
@@ -221,7 +224,7 @@ autospecnew: preautospecnew-checks localreponotice
 			--name $(NAME) \
 			--archives $(ARCHIVES) \
 			--mock-config $(MOCK_CONFIG_VAL) \
-			--mock-opts "$(MOCK_OPTS)" \
+			--mock-opts="$(MOCK_OPTS)" \
 			$${SETVERSION:+ --version $${SETVERSION}} \
 			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 			$(URL); \

README-advanced.md

@@ -0,0 +1,40 @@
+## Advanced topics
+
+### Manual setup
+
+If you did not run the user-setup script (see "Automated setup" section in the
+[main README](README.md), you will want to set up the developer tooling
+workspace manually. This section provides general documentation for the manual
+setup process, and it is not meant to be exhaustive.
+
+On your Clear Linux system, create a workspace for Clear Linux development
+work:
+
+```
+$ mkdir clearlinux
+```
+
+Clone this repo into a `projects` directory within the workspace:
+
+```
+$ cd clearlinux
+$ mkdir projects
+$ git clone https://github.com/clearlinux/common projects/common
+```
+
+Create the toplevel tooling Makefile:
+
+```
+$ ln -s projects/common/Makefile.toplevel Makefile
+```
+
+Clone all Clear Linux package and project repositories:
+
+```
+$ make clone
+```
+
+Note: You can clone the repos in parallel by using make's `-j` option.
+
+At this point, the `packages` directory will contain all Clear Linux package
+repos, and `projects` will contain common, clr-bundles, and autospec repos.

README.md

@@ -152,39 +152,4 @@ If `Makefile.config.site_local` doesn't exist already, create it.
 
 ### Manual setup
 
-If you did not run the user-setup script (see "Automated setup" section above),
-you will want to set up the developer tooling workspace manually. This section
-provides general documentation for the manual setup process, and it is not
-meant to be exhaustive.
-
-On your Clear Linux system, create a workspace for Clear Linux development
-work:
-
-```
-$ mkdir clearlinux
-```
-
-Clone this repo into a `projects` directory within the workspace:
-
-```
-$ cd clearlinux
-$ mkdir projects
-$ git clone https://github.com/clearlinux/common projects/common
-```
-
-Create the toplevel tooling Makefile:
-
-```
-$ ln -s projects/common/Makefile.toplevel Makefile
-```
-
-Clone all Clear Linux package and project repositories:
-
-```
-$ make clone
-```
-
-Note: You can clone the repos in parallel by using make's `-j` option.
-
-At this point, the `packages` directory will contain all Clear Linux package
-repos, and `projects` will contain common, clr-bundles, and autospec repos.
+See the [Manual setup](README-advanced.md#manual-setup) documentation.

licenses-exceptions

@@ -1,4 +1,5 @@
 Copyright
 Distributable
+ISSL
 Public-Domain
 sendmail

licenses-spdx

@@ -237,6 +237,7 @@ MTLL
 MakeIndex
 MirOS
 Motosoto
+MulanPSL-1.0
 Multics
 Mup
 NASA-1.3
@@ -251,6 +252,7 @@ NPL-1.1
 NPOSL-3.0
 NRL
 NTP
+NTP-0
 Naumen
 Net-SNMP
 NetCDF
@@ -263,7 +265,12 @@ OCLC-2.0
 ODC-By-1.0
 ODbL-1.0
 OFL-1.0
+OFL-1.0-RFN
+OFL-1.0-no-RFN
 OFL-1.1
+OFL-1.1-RFN
+OFL-1.1-no-RFN
+OGL-Canada-2.0
 OGL-UK-1.0
 OGL-UK-2.0
 OGL-UK-3.0
@@ -296,6 +303,7 @@ OpenSSL
 PDDL-1.0
 PHP-3.0
 PHP-3.01
+PSF-2.0
 Parity-6.0.0
 Plexus
 PostgreSQL
@@ -323,6 +331,8 @@ SMLNJ
 SMPPL
 SNIA
 SPL-1.0
+SSH-OpenSSH
+SSH-short
 SSPL-1.0
 SWL
 Saxpath
@@ -343,6 +353,7 @@ TORQUE-1.1
 TOSL
 TU-Berlin-1.0
 TU-Berlin-2.0
+UCL-1.0
 UPL-1.0
 Unicode-DFS-2015
 Unicode-DFS-2016
@@ -382,10 +393,12 @@ diffmark
 dvipdfm
 eCos-2.0
 eGenix
+etalab-2.0
 gSOAP-1.3b
 gnuplot
 iMatix
 libpng-2.0
+libselinux-1.0
 libtiff
 mpich2
 psfrag

mapping

@@ -51,4 +51,5 @@ rack = rubygem-rack
 compute = nova
 international_components_for_unicode = icu4c
 node.js = nodejs
-jupyter-core = jupyter_core
+jupyter-core = jupyter_core
+flatten-json = flatten_json

patchfilter.py

@@ -102,6 +102,22 @@ def zap_entire_file_end(filename):
         if file.endswith(filename):
             files.remove(file)                
             
+def zap_line_in_file_substring(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_delete = list()
+                for line in chunk:
+                    if match in line:
+                        to_delete.append(line)
+                for line in to_delete:
+                    chunk.remove(line)
+            
+
 def zap_line_in_file(filename, match):
     global header
     global files
@@ -209,6 +225,11 @@ def main():
     zap_entire_file_end("xz.sig")
     zap_entire_file_end("gz.sig")
     zap_entire_file_end("bz2.sig")
+    zap_entire_file_end("xz.asc")
+    zap_entire_file_end("gz.asc")
+    zap_entire_file_end("bz2.asc")
+    
+    zap_entire_file_end(".pkey")
     
     zap_line_in_file("b/testresults", "Total : 0")
     zap_line_in_file("b/testresults", "Pass : 0")
@@ -223,14 +244,15 @@ def main():
     
     zap_line_in_file_start(".spec", "Release  :")
     zap_line_in_file_start(".spec", "Source99 :")
-    zap_line_in_file_start(".spec", "Source0  :")
     zap_line_in_file_start(".spec", "export SOURCE_DATE_EPOCH")
     zap_line_in_file_start(".spec", "export AR=gcc-ar")
     zap_line_in_file_start(".spec", "export RANLIB=gcc-ranlib")
     zap_line_in_file_start(".spec", "export NM=gcc-nm")
     zap_line_in_file_start(".spec", "export CFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
     zap_line_in_file_start(".spec", "export FCFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FCFLAGS=\"$FFLAGS -O3 -ffat-lto-objects -flto=4 \"")
     zap_line_in_file_start(".spec", "export FFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FFLAGS=\"$FFLAGS -O3 -ffat-lto-objects -flto=4 \"")
     zap_line_in_file_start(".spec", "export CXXFLAGS=\"$CXXFLAGS -O3 -ffat-lto-objects -flto=4 \"")
     zap_line_in_file_start(".spec", "%setup -q -n ")
     zap_line_in_file_start(".spec", "URL      :")
@@ -265,12 +287,17 @@ def main():
     zap_line_in_file_start(".spec", "%description license")
     zap_line_in_file_start(".spec", "%package doc")
     zap_line_in_file_start(".spec", "cp -a ")
+    zap_line_in_file_start(".spec", "cd %{_builddir}/")
+    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
+    zap_line_in_file_start(".spec", "rm -rf %{buildroot}")
  
     zap_line_in_file_start("b/Makefile", "URL =")
     zap_line_in_file_start("b/Makefile", "PKG_NAME :=")
 
+    zap_line_in_file_start("b/options.conf", "name = ")
     zap_line_in_file("b/options.conf", "archives = ")
     zap_line_in_file("b/options.conf", "giturl = ")
+    zap_line_in_file_start("b/options.conf", "url = ")
     zap_line_in_file("b/options.conf", "[package]")
     zap_line_in_file("b/options.conf", "")
     zap_line_in_file("b/options.conf", "[autospec]")
@@ -287,11 +314,13 @@ def main():
     zap_line_in_file("b/options.conf", "# disable parallelization during build")
     zap_line_in_file("b/options.conf", "broken_parallel_build = false")
     zap_line_in_file("b/options.conf", "# this package is a library compatability package and only ships versioned library files")
+    zap_line_in_file("b/options.conf", "# this package is a library compatibility package and only ships versioned library files")
     zap_line_in_file("b/options.conf", "compat = false")
     zap_line_in_file("b/options.conf", "# set conservative build flags")
     zap_line_in_file("b/options.conf", "conservative_flags = false")
     zap_line_in_file("b/options.conf", "# dev package requires the extras to be installed")
     zap_line_in_file("b/options.conf", "dev_requires_extras = false")
+    zap_line_in_file("b/options.conf", "nodebug = false")
     zap_line_in_file("b/options.conf", "# pass -ffast-math to compiler")
     zap_line_in_file("b/options.conf", "fast-math = false")
     zap_line_in_file("b/options.conf", "# optimize build for speed over size")
@@ -321,9 +350,17 @@ def main():
     zap_line_in_file("b/options.conf", "# add clang flags")
     zap_line_in_file("b/options.conf", "use_clang = false")
     zap_line_in_file("b/options.conf", "# configure build for lto")
-    zap_line_in_file("b/options.conf", "use_lto = false")
+    zap_line_in_file("b/options.conf", "use_lto = true")
     zap_line_in_file("b/options.conf", "# require package verification for build")
     zap_line_in_file("b/options.conf", "verify_required = true")
+    zap_line_in_file("b/options.conf", "# do not generate debuginfo for this package")
+    zap_line_in_file("b/options.conf", "# configure build also for openmpi")
+    zap_line_in_file("b/options.conf", "openmpi = false")
+    zap_line_in_file("b/options.conf", "alias =")
+    zap_line_in_file("b/options.conf", "alias = ")
+    zap_line_in_file("b/options.conf", "nodebug = false")
+    zap_line_in_file("b/options.conf", "domain =")
+    zap_line_in_file("b/options.conf", "domain = ")
 
     zap_line_in_file("b/buildreq_add", "# This file contains additional build requirements that did not get")
     zap_line_in_file("b/buildreq_add", "# picked up automatically. One name per line, no whitespace.")
@@ -349,7 +386,11 @@ def main():
     zap_plus_line_in_file("symbols32")
 
     zap_entire_file("b/whatrequires")
+    zap_entire_file("b/versions")
+    zap_entire_file("b/.gitignore")
+    
 
+    zap_line_in_file_substring(".spec", "%{buildroot}/usr/share/package-licenses")   # version number change in license copy
     zap_empty_chunks()
     
     print_all()

user-setup.sh

@@ -102,7 +102,7 @@ fi
 
 required_progs() {
   local bindir="/usr/bin"
-  for f in git mock rpm rpmbuild ; do
+  for f in git mock rpm rpmbuild diffstat ; do
     [ ! -x "${bindir}/${f}" ] && missing+="${f} "
   done
   [ "$PEM" ] && [ ! -x /usr/bin/koji ] && missing+="koji "