Make catchup- a little bit more flexible with tag names

We support tag names like
${package}-${version}
${package}-v${version}
${version}
v${version}

Now add support for things like
${package}-foo-bar-${version}
foo-bar-${version}

for example:
vulkan-sdk-1.3.275.0 for SPIRV-Headers

Makefile.common

@@ -166,6 +166,7 @@ preautospec-checks:
 #help to autospec.
 #help For more information, see the project at https://github.com/clearlinux/autospec
 autospec: preautospec-checks pullrebase localreponotice clean-old-content
+	git rm --ignore-unmatch pumpAutospec || rm -f pumpAutospec
 	@if [ -e $(SPECFILE) ] && ! grep -q "# Generated by: autospec.py" $(SPECFILE) ; then \
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
@@ -222,6 +223,7 @@ scanlicense:
 #help the variable BUMP_MSG is set, its value is used as the commit summary.
 #help Otherwise a generic commit summary is used.
 bump:
+	git stash
 	git pull --rebase
 	$(MAKE) bumpnogit
 	git add $(SPECFILE) release
@@ -406,6 +408,7 @@ update-versions:
 #help or update the package and push the update to koji.
 #help If no update.sh exists, update the version, autospec and push the
 #help update to koji (bumping the bump.list if found).
+update: export AUTOSPEC_UPDATE=1
 update:
 	@if [ -f update.first ]; then \
 		for pkg in $$(< update.first); do \
@@ -579,6 +582,78 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+#help catchup: Backport the commits from the current version to the upstream HEAD (not release).
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup:
+	$(MAKE) catchup-HEAD
+
+#help catchup-<commit|tag>: Backport the commits from the current version to the specified commit or tag.
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup-%:
+	@target=$*; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	if ! git -C results/$(PKG_NAME) rev-parse --verify --quiet "$${target}" >/dev/null; then \
+		echo "Error: Target commit/tag $${target} not found"; \
+		exit 1; \
+	fi; \
+	version=$$(rpm -q --qf '%{VERSION}\n' --specfile $(SPECFILE) | head -1); \
+	echo "Version: $${version}"; \
+	current_tag=$$(git -C results/$(PKG_NAME) tag --list | grep -E "^($(PKG_NAME)-)?(.+-|v)?$${version}$$") || { \
+		echo "Error: No tag found for current package version"; \
+		exit 1; \
+	}; \
+	echo "Catching up from $${current_tag} to $${target}"; \
+	for commit in $$(git -C results/$(PKG_NAME) log --reverse --pretty=oneline $${current_tag}..$${target} | cut -d' ' -f1); do \
+		$(MAKE) backport-$${commit}; \
+	done;
+
+
+#help backport-<commit>: Retrieve a commit from the upstream git repository and save it as a backport patch.
+#help The giturl is read from options.conf.
+backport-%:
+	@commit=$*; \
+	echo "Backporting commit: $${commit}"; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	full_commit=$$(git -C results/$(PKG_NAME) show --pretty=oneline $${commit} 2>/dev/null | head -1 | cut -d' ' -f 1); \
+	if [[ -z "$${full_commit}" ]]; then \
+		echo "Error: Commit for $${commit} not found"; \
+		exit 1; \
+	fi; \
+	patch=backport-$${full_commit}.patch; \
+	git -C results/$(PKG_NAME) format-patch -1 --stdout $${full_commit} > $${patch}; \
+	if [[ -s $${patch} ]]; then \
+		echo "$${patch} created"; \
+		grep -qE "^$${patch}$$" series 2>/dev/null || echo "$${patch}" >> series; \
+	else \
+		rm -f $${patch}; \
+		echo "Error: Failed to create backport patch"; \
+		exit 1; \
+	fi \
+
 .PHONY: whatrequires
 #help whatrequires: Output a list of packages that directly depend on this one,
 #help showing the subpackage-level breakdown. Each line of output has the format

licenses-extra

@@ -5,3 +5,4 @@ GFDL-1.3+
 MIT-Opengroup
 WXwindows
 w3c
+libpng-2.0

vendor.py

@@ -9,9 +9,8 @@ import subprocess
 import tempfile
 import time
 
-from git import Repo
-
 import requests
+from git import Repo
 
 
 def vendor_check():
@@ -51,16 +50,17 @@ def setup_content(url):
 
 
 def setup_cargo_vendor(path):
+    cargo_paths = []
     for dirpath, _, files in os.walk(path):
         for fname in files:
             if fname == "Cargo.toml":
-                return dirpath
-    return False
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
 
 
-def update_cargo_vendor(path, name, git):
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
     git_uri = os.path.join(git, name)
-    vendor_path = os.path.join(path, 'vendor')
+    vendor_path = os.path.join(tmpdir, 'vendor')
     subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
                    stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
@@ -70,17 +70,26 @@ def update_cargo_vendor(path, name, git):
                        stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
                        shell=True, check=True, stdout=subprocess.DEVNULL)
-    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
-    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    subprocess.run('cargo vendor', cwd=path, shell=True, check=True,
-                   stdout=subprocess.DEVNULL)
-    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
+                                   check=True, stdout=subprocess.PIPE,
+                                   universal_newlines=True).stdout
+    with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
+        gafile.write("* text=false\n")
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
-        return False
+        # Always use the newest tag as sometimes a new tag will
+        # be created but the package won't be updated to use it
+        # for a different failure reason.
+        tag = sorted(repo.tags, key=lambda x: x.name, reverse=True)[0]
+        return tag, cargo_vendors
     subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
                    stdout=subprocess.DEVNULL)
     subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
@@ -92,10 +101,10 @@ def update_cargo_vendor(path, name, git):
     subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     time.sleep(30)
-    return tag
+    return tag, cargo_vendors
 
 
-def update_cargo_sources(name, tag):
+def update_cargo_sources(name, tag, cargo_vendors):
     makefile = []
     options = []
     archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
@@ -105,11 +114,11 @@ def update_cargo_sources(name, tag):
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_replace}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
                 print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
@@ -122,20 +131,21 @@ def update_cargo_sources(name, tag):
     with open('options.conf', encoding='utf8') as ofile:
         for line in ofile.readlines():
             if line.startswith('archives'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
                 options.append(new_archives)
             else:
                 options.append(line)
     with open('options.conf', 'w', encoding='utf8') as ofile:
         ofile.writelines(options)
+    with open('cargo_vendors', 'w', encoding='utf8') as cfile:
+        cfile.write(cargo_vendors)
 
 
 def main():
-    updated = False
     args = get_args()
 
     vtype = vendor_check()
@@ -145,14 +155,12 @@ def main():
 
     tdir = setup_content(args.url)
     if vtype == 'cargo':
-        vdir = setup_cargo_vendor(tdir)
-        if vdir:
-            tag = update_cargo_vendor(vdir, args.name, args.git)
-            if tag:
-                update_cargo_sources(args.name, tag)
-                updated = True
-    if not updated:
-        print(args.archives)
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
+            print(args.archives)
+        else:
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
+            update_cargo_sources(args.name, tag, cargo_vendors)
     shutil.rmtree(tdir)