update packages file

Changes vendor from a target (it could still have a stand-alone target
but it wouldn't be used by the autospec target).

This change is needed because the Makefile is rewritten in the
autospec target but the ARCHIVES make variable is going to use the
value when make is executed. The common change causes the ARCHIVES
value to be eval'd during the autospec target execution in order for
the update from the vendor script to be seen by make.

The vendor script changes fix bugs in updating the options.conf and
Makefile. It also now handles returning the original ARCHIVES value as
well as prevent subshell scripts from outputting and adding badness
into the eval'd ARCHIVES variable.

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -170,6 +170,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
@@ -386,6 +387,17 @@ koji-waitrepo: kojidef
 		koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
 	fi
 
+.PHONY: koji-tag
+#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
+koji-tag:
+	koji tag-pkg dist-clear $(SRPMVERS)
+
+.PHONY: koji-untag
+#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
+koji-untag:
+	koji untag-pkg dist-clear $(SRPMVERS)
+
+
 update-versions:
 
 .PHONY: update

Makefile.config

@@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs
 # Location where project repos are hosted.
 PRJ_BASE_URL = https://github.com/clearlinux
 
+# Location where vendor repos are hosted.
+VND_BASE_URL =
+
 # Project repos that are integral to the common tooling.
 PROJ_PKGS = autospec clr-bundles common
 

Makefile.toplevel

@@ -52,7 +52,7 @@ $(clone_PKGS): $(PACKAGES_FILE) packages/common/Makefile.common
 	remotepkg=$(call remotepkgname,$$pkg); \
 	[ -d packages/$$pkg ] || ( \
 	echo "Checking out: packages/$$pkg"; \
-	git clone $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
+	git clone -b main $(PKG_BASE_URL)/$$remotepkg packages/$$pkg; \
 	cd packages/$$pkg; \
 	$(call gitoliteurl,packages/$$pkg); \
 	$(call subjectprefix,$$pkg); \
@@ -126,11 +126,17 @@ endif
 pull: ${PULL_PKGS}
 
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
-#help removes the associated package repos from the ./packages tree
+#help removes the associated package repos from the ./packages tree and stores
+#help it in packages_old. If it already exists in packages_old, skip it.
 clean-pkgs-dir: $(PACKAGES_FILE)
 	@for p in ${PKGS_ORPHANED}; do \
-		echo "Removing $$p from packages, it is no longer in common/packages."; \
-		rm -rf packages/$$p; \
+		mkdir -p packages_old; \
+		if [ -d "packages_old/$$p" ]; then \
+			echo "packages_old/$$p already exists, not touching it"; \
+		else \
+			echo "Moving $$p from packages to packages_old, as it is no longer in common/packages."; \
+			mv packages/$$p packages_old/$$p; \
+		fi; \
 	done
 	@echo "packages directory cleaned";
 

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

licenses-spdx

@@ -180,6 +180,7 @@ FDK-AAC
 FSFAP
 FSFUL
 FSFULLR
+FSFULLRWD
 FTL
 Fair
 Frameworx-1.0
@@ -252,6 +253,7 @@ JPNIC
 JSON
 Jam
 JasPer-2.0
+Knuth-CTAN
 LAL-1.2
 LAL-1.3
 LGPL-2.0
@@ -471,6 +473,7 @@ Zlib
 blessing
 bzip2-1.0.5
 bzip2-1.0.6
+checkmk
 copyleft-next-0.3.0
 copyleft-next-0.3.1
 curl
@@ -485,6 +488,7 @@ iMatix
 libpng-2.0
 libselinux-1.0
 libtiff
+libutil-David-Nugent
 mpi-permissive
 mpich2
 mplus

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:

vendor.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+from git import Repo
+
+import requests
+
+
+def vendor_check():
+    if not os.path.isfile('options.conf'):
+        return False
+    config = configparser.ConfigParser(interpolation=None)
+    config.read('options.conf')
+    if 'autospec' not in config.sections():
+        return False
+    if vendor := config['autospec'].get('cargo_vendor'):
+        if vendor == "true":
+            return 'cargo'
+
+    return False
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
+    parser.add_argument('url')
+    parser.add_argument('name')
+    parser.add_argument('git')
+    return parser.parse_args()
+
+
+def setup_content(url):
+    tdir = tempfile.mkdtemp()
+    outfile = os.path.join(tdir, os.path.basename(url))
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()
+    with open(outfile, 'wb') as cfile:
+        cfile.write(response.content)
+
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
+    os.remove(outfile)
+    return tdir
+
+
+def setup_cargo_vendor(path):
+    for dirpath, _, files in os.walk(path):
+        for fname in files:
+            if fname == "Cargo.toml":
+                return dirpath
+    return False
+
+
+def update_cargo_vendor(path, name, git):
+    git_uri = os.path.join(git, name)
+    vendor_path = os.path.join(path, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    vendor_git = os.path.join(vendor_path, '.git')
+    if not os.path.isdir(vendor_git):
+        # initialize a git repo
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
+        subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
+    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    shutil.rmtree(vendor_path)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    repo = Repo(vendor_path)
+    if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
+        return False
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
+    subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    gmt = time.gmtime()
+    tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
+    subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
+                   check=True, stdout=subprocess.DEVNULL)
+    subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
+    time.sleep(30)
+    return tag
+
+
+def update_cargo_sources(name, tag):
+    makefile = []
+    options = []
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
+                                 'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
+    with open('Makefile', encoding='utf8') as mfile:
+        for line in mfile.readlines():
+            if line.startswith('ARCHIVES'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_replace}-{tag}.tar.xz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
+                makefile.append(new_archives)
+            else:
+                makefile.append(line)
+    with open('Makefile', 'w', encoding='utf8') as mfile:
+        mfile.writelines(makefile)
+
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+
+
+def main():
+    updated = False
+    args = get_args()
+
+    vtype = vendor_check()
+    if not vtype:
+        print(args.archives)
+        return
+
+    tdir = setup_content(args.url)
+    if vtype == 'cargo':
+        vdir = setup_cargo_vendor(tdir)
+        if vdir:
+            tag = update_cargo_vendor(vdir, args.name, args.git)
+            if tag:
+                update_cargo_sources(args.name, tag)
+                updated = True
+    if not updated:
+        print(args.archives)
+    shutil.rmtree(tdir)
+
+
+if __name__ == '__main__':
+    main()