Don't blacklist the update target directory

This is shipped by the filesystem package.

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -20,7 +20,7 @@ LATEST_RPMS = $(wildcard rpms/*.rpm)
 DEBUGINFO_RPM = $(wildcard rpms/${PKG_NAME}-debuginfo-$(shell $(call queryspec,%{V}-%{R},${SPECFILE})).*.rpm)
 RPMS ?= $(LATEST_RPMS)
 
-WITH_SUDO = $(shell cmp /usr/bin/mock /usr/sbin/mock &> /dev/null && echo sudo )
+WITH_SUDO = sudo
 
 # MOCK_CONFIG_VAL is set in Makefile.shared
 MOCK_SMP_FLAGS=$(patsubst -j%, -D_smp_mflags\ -j%,$(filter -j%, $(MAKEFLAGS)))
@@ -157,6 +157,10 @@ pullrebase:
 		fi \
 	fi
 
+# help vendor: Attempt to create an updated vendor tar archive if needed
+vendor: preautospec-checks pullrebase clean-old-content
+	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
+
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -171,6 +175,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		exit 1; \
 	fi
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
+	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \
@@ -217,14 +222,24 @@ autospecnostate:
 scanlicense:
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py -t . --config $(AUTOSPEC_CONF) --license-only $(firstword $(NEWURL) $(URL)) --name $(PKG_NAME)
 
-#help bump: Increments the release file by one.
+#help bump: Increments the package release number by one and commits the result. If
+#help the variable BUMP_MSG is set, its value is used as the commit summary.
+#help Otherwise a generic commit summary is used.
 bump:
 	git pull --rebase
 	$(MAKE) bumpnogit
 	git add $(SPECFILE) release
-	git commit -a -m "version bump from $(shell $(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))) to `$(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))`"
+	fromver=$(shell $(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))); \
+	tover=`$(call queryspec,%{VERSION}-%{RELEASE}\n,$(SPECFILE))`; \
+	if [[ -n "$(BUMP_MSG)" ]]; then \
+		bumpmsg="$(BUMP_MSG)"; \
+	else \
+		bumpmsg="version bump from $$fromver to $$tover"; \
+	fi; \
+	git commit -a -m "$$bumpmsg"
 
-#help bumpnogit: Increments the release file by one and does not commit changes.
+#help bumpnogit: Increments the package release number by one and does not commit
+#help changes.
 bumpnogit:
 	@$(MAKE) spdxcheck
 	oldrel=$(shell $(call queryspec,%{RELEASE}\n,$(SPECFILE))); \
@@ -331,8 +346,9 @@ koji: prekoji-checks kojidef
 	$(KOJI_CMD) build $$KOJI_NOWAIT $(KOJI_TAG) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)
 	@if [ -f bump.list ]; then \
 		$(MAKE) koji-waitrepo; \
+		msg="Bump for $(shell $(call queryspec,%{NAME} %{VERSION}\n,$(SPECFILE)))"; \
 		for bump_dep in $$(cat bump.list); do \
-			(cd ../"$$bump_dep" && $(MAKE) bump && $(MAKE) koji-nowait) \
+			(cd ../"$$bump_dep" && $(MAKE) bump BUMP_MSG="$$msg" && $(MAKE) koji-nowait) \
 		done \
 	fi
 
@@ -375,6 +391,17 @@ koji-waitrepo: kojidef
 		koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
 	fi
 
+.PHONY: koji-tag
+#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
+koji-tag:
+	koji tag-pkg dist-clear $(SRPMVERS)
+
+.PHONY: koji-untag
+#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
+koji-untag:
+	koji untag-pkg dist-clear $(SRPMVERS)
+
+
 update-versions:
 
 .PHONY: update

Makefile.config

@@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs
 # Location where project repos are hosted.
 PRJ_BASE_URL = https://github.com/clearlinux
 
+# Location where vendor repos are hosted.
+VND_BASE_URL =
+
 # Project repos that are integral to the common tooling.
 PROJ_PKGS = autospec clr-bundles common
 

Makefile.toplevel

@@ -52,7 +52,7 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	remotepkg=$(call remotepkgname,$$pkg); \
 	[ -d packages/$$pkg ] || ( \
 	echo "Checking out: packages/$$pkg"; \
-	git clone $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
+	git clone -b main $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
 	cd packages/$$pkg; \
 	$(call gitoliteurl,packages/$$pkg); \
 	$(call subjectprefix,$$pkg); \
@@ -126,11 +126,17 @@ endif
 pull: ${PULL_PKGS}
 
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
-#help removes the associated package repos from the ./packages tree
+#help removes the associated package repos from the ./packages tree and stores
+#help it in packages_old. If it already exists in packages_old, skip it.
 clean-pkgs-dir: $(PACKAGES_FILE)
 	@for p in ${PKGS_ORPHANED}; do \
-		echo "Removing $$p from packages, it is no longer in common/packages."; \
-		rm -rf packages/$$p; \
+		mkdir -p packages_old; \
+		if [ -d "packages_old/$$p" ]; then \
+			echo "packages_old/$$p already exists, not touching it"; \
+		else \
+			echo "Moving $$p from packages to packages_old, as it is no longer in common/packages."; \
+			mv packages/$$p packages_old/$$p; \
+		fi; \
 	done
 	@echo "packages directory cleaned";
 
@@ -168,7 +174,7 @@ status: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 #help the --stat flag.
 diff: $(PACKAGES_FILE) $(addprefix packages/,$(PKGS_LOCAL))
 	@for p in projects/common $(addprefix packages/,$(PKGS_LOCAL)); do \
-		(cd $$p; git status --porcelain | grep -vq '^?? ' && (echo "Uncommitted changes in: $$p"; git diff --stat); :) ;\
+		(cd $$p; git status --porcelain | grep -vq '^?? ' && (echo "Uncommitted changes in: $$p"; git --no-pager diff --stat); :) ;\
 	done
 
 #help log: Run git log origin/main..HEAD for all package repos, thus

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

licenses-spdx

@@ -32,6 +32,7 @@ Apache-1.0
 Apache-1.1
 Apache-2.0
 App-s2p
+Arphic-1999
 Artistic-1.0
 Artistic-1.0-Perl
 Artistic-1.0-cl8
@@ -59,11 +60,13 @@ BSD-Protection
 BSD-Source-Code
 BSL-1.0
 BUSL-1.1
+Baekmuk
 Bahyph
 Barr
 Beerware
 BitTorrent-1.0
 BitTorrent-1.1
+Bitstream-Vera
 BlueOak-1.0.0
 Borceux
 C-UDA-1.0
@@ -77,6 +80,7 @@ CC-BY-2.5-AU
 CC-BY-3.0
 CC-BY-3.0-AT
 CC-BY-3.0-DE
+CC-BY-3.0-IGO
 CC-BY-3.0-NL
 CC-BY-3.0-US
 CC-BY-4.0
@@ -176,6 +180,7 @@ FDK-AAC
 FSFAP
 FSFUL
 FSFULLR
+FSFULLRWD
 FTL
 Fair
 Frameworx-1.0
@@ -248,6 +253,7 @@ JPNIC
 JSON
 Jam
 JasPer-2.0
+Knuth-CTAN
 LAL-1.2
 LAL-1.3
 LGPL-2.0
@@ -270,6 +276,8 @@ LPPL-1.1
 LPPL-1.2
 LPPL-1.3a
 LPPL-1.3c
+LZMA-SDK-9.11-to-9.20
+LZMA-SDK-9.22
 Latex2e
 Leptonica
 LiLiQ-P-1.1
@@ -291,10 +299,12 @@ MPL-1.0
 MPL-1.1
 MPL-2.0
 MPL-2.0-no-copyleft-exception
+MS-LPL
 MS-PL
 MS-RL
 MTLL
 MakeIndex
+Minpack
 MirOS
 Motosoto
 MulanPSL-1.0
@@ -307,6 +317,7 @@ NBPL-1.0
 NCGL-UK-2.0
 NCSA
 NGPL
+NICTA-1.0
 NIST-PD
 NIST-PD-fallback
 NLOD-1.0
@@ -381,6 +392,7 @@ PolyForm-Noncommercial-1.0.0
 PolyForm-Small-Business-1.0.0
 PostgreSQL
 Python-2.0
+Python-2.0.1
 QPL-1.0
 Qhull
 RHeCos-1.1
@@ -461,6 +473,7 @@ Zlib
 blessing
 bzip2-1.0.5
 bzip2-1.0.6
+checkmk
 copyleft-next-0.3.0
 copyleft-next-0.3.1
 curl
@@ -475,7 +488,10 @@ iMatix
 libpng-2.0
 libselinux-1.0
 libtiff
+libutil-David-Nugent
+mpi-permissive
 mpich2
+mplus
 psfrag
 psutils
 wxWindows

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:

vendor.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+from git import Repo
+
+import requests
+
+
+def vendor_check():
+    if not os.path.isfile('options.conf'):
+        return False
+    config = configparser.ConfigParser(interpolation=None)
+    config.read('options.conf')
+    if 'autospec' not in config.sections():
+        return False
+    if vendor := config['autospec'].get('cargo_vendor'):
+        if vendor == "true":
+            return 'cargo'
+
+    return False
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('url')
+    parser.add_argument('name')
+    parser.add_argument('git')
+    return parser.parse_args()
+
+
+def setup_content(url):
+    tdir = tempfile.mkdtemp()
+    outfile = os.path.join(tdir, os.path.basename(url))
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()
+    with open(outfile, 'wb') as cfile:
+        cfile.write(response.content)
+
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    os.remove(outfile)
+    return tdir
+
+
+def setup_cargo_vendor(path):
+    for dirpath, _, files in os.walk(path):
+        for fname in files:
+            if fname == "Cargo.toml":
+                return dirpath
+    return False
+
+
+def update_cargo_vendor(path, name, git):
+    git_uri = os.path.join(git, name)
+    vendor_path = os.path.join(path, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    vendor_git = os.path.join(vendor_path, '.git')
+    if not os.path.isdir(vendor_git):
+        # initialize a git repo
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
+                       shell=True, check=True)
+    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+                   shell=True, check=True)
+    shutil.rmtree(vendor_path)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+                   shell=True, check=True)
+    repo = Repo(vendor_path)
+    if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
+        return False
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
+                   shell=True, check=True)
+    gmt = time.gmtime()
+    tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
+    subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
+                   check=True)
+    subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
+                   shell=True, check=True)
+    time.sleep(30)
+    return tag
+
+
+def update_cargo_sources(name, tag):
+    makefile = []
+    options = []
+    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                 'snapshot', name)
+    with open('Makefile', encoding='utf8') as mfile:
+        for line in mfile.readlines():
+            if line.startswith('ARCHIVES'):
+                if re.match(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                else:
+                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                makefile.append(new_archives)
+            else:
+                makefile.append(line)
+    with open('Makefile', 'w', encoding='utf8') as mfile:
+        mfile.writelines(makefile)
+
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.match(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                else:
+                    new_archives = f"achives = {archive_match}-{tag}.tar.xz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+
+
+def main():
+    vtype = vendor_check()
+    if not vtype:
+        return
+
+    args = get_args()
+    tdir = setup_content(args.url)
+    if vtype == 'cargo':
+        vdir = setup_cargo_vendor(tdir)
+        if vdir:
+            tag = update_cargo_vendor(vdir, args.name, args.git)
+            if tag:
+                update_cargo_sources(args.name, tag)
+    shutil.rmtree(tdir)
+
+if __name__ == '__main__':
+    main()