update packages file

Make sure we only parse %{version} one time while doing a catchup.

Makefile.common

@@ -582,6 +582,78 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+#help catchup: Backport the commits from the current version to the upstream HEAD (not release).
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup:
+	$(MAKE) catchup-HEAD
+
+#help catchup-<commit|tag>: Backport the commits from the current version to the specified commit or tag.
+#help Only works if giturl is defined and the current package version can be mapped to a git tag.
+catchup-%:
+	@target=$*; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	if ! git -C results/$(PKG_NAME) rev-parse --verify --quiet "$${target}" >/dev/null; then \
+		echo "Error: Target commit/tag $${target} not found"; \
+		exit 1; \
+	fi; \
+	version=$$(rpm -q --qf '%{VERSION}\n' --specfile $(SPECFILE) | head -1); \
+	echo "Version: $${version}"; \
+	current_tag=$$(git -C results/$(PKG_NAME) tag --list | grep -E "^($(PKG_NAME)-)?v?$${version}$$") || { \
+		echo "Error: No tag found for current package version"; \
+		exit 1; \
+	}; \
+	echo "Catching up from $${current_tag} to $${target}"; \
+	for commit in $$(git -C results/$(PKG_NAME) log --reverse --pretty=oneline $${current_tag}..$${target} | cut -d' ' -f1); do \
+		$(MAKE) backport-$${commit}; \
+	done;
+
+
+#help backport-<commit>: Retrieve a commit from the upstream git repository and save it as a backport patch.
+#help The giturl is read from options.conf.
+backport-%:
+	@commit=$*; \
+	echo "Backporting commit: $${commit}"; \
+	giturl=$$(grep -E '^giturl\s*=\s*\S+' options.conf | sed 's/giturl\s*=\s*//' 2>/dev/null); \
+	if [[ -z "$${giturl}" ]]; then \
+		echo "Error: giturl not defined in options.conf"; \
+		exit 1; \
+	fi; \
+	mkdir -p results; \
+	if [[ -d results/$(PKG_NAME) ]]; then \
+		echo "Reusing existing repository..."; \
+		git -C results/$(PKG_NAME) fetch origin; \
+	else \
+		echo "Cloning upstream repository..."; \
+		git -C results clone "$${giturl}" $(PKG_NAME); \
+	fi; \
+	full_commit=$$(git -C results/$(PKG_NAME) show --pretty=oneline $${commit} 2>/dev/null | head -1 | cut -d' ' -f 1); \
+	if [[ -z "$${full_commit}" ]]; then \
+		echo "Error: Commit for $${commit} not found"; \
+		exit 1; \
+	fi; \
+	patch=backport-$${full_commit}.patch; \
+	git -C results/$(PKG_NAME) format-patch -1 --stdout $${full_commit} > $${patch}; \
+	if [[ -s $${patch} ]]; then \
+		echo "$${patch} created"; \
+		grep -qE "^$${patch}$$" series 2>/dev/null || echo "$${patch}" >> series; \
+	else \
+		rm -f $${patch}; \
+		echo "Error: Failed to create backport patch"; \
+		exit 1; \
+	fi \
+
 .PHONY: whatrequires
 #help whatrequires: Output a list of packages that directly depend on this one,
 #help showing the subpackage-level breakdown. Each line of output has the format

packages

@@ -180,6 +180,7 @@ R-Ryacas
 R-SGP
 R-SGPdata
 R-SQUAREM
+R-SimDesign
 R-Sleuth2
 R-SnowballC
 R-SparseM
@@ -232,6 +233,7 @@ R-bayesplot
 R-bazar
 R-bbmle
 R-bdsmatrix
+R-beepr
 R-beeswarm
 R-bench
 R-benchr
@@ -871,6 +873,7 @@ R-rsvg
 R-runjags
 R-rversions
 R-rvest
+R-rworldmap
 R-s2
 R-sampling
 R-sandwich
@@ -1238,6 +1241,7 @@ byobu
 bz2file
 bzip2
 c-ares
+c-blosc
 c-blosc2
 cJSON
 c_rehash
@@ -1339,6 +1343,7 @@ compat-cairomm-soname10
 compat-codec2-soname1
 compat-enchant-soname1
 compat-ffmpeg-4.4
+compat-ffmpeg-6
 compat-fuse-soname2
 compat-gcc-10
 compat-gcr-soname1
@@ -1360,6 +1365,7 @@ compat-libvpx-soname8
 compat-pangomm-soname14
 compat-protobuf-soname29
 compat-protobuf-soname32
+compat-python3-rolling
 compat-re2-soname10
 compat-readline-soname5
 compat-taglib-soname1
@@ -1401,6 +1407,7 @@ cups-filters
 cups-pk-helper
 curl
 cycler
+cyme
 cyrus-sasl
 dapl
 darktable
@@ -1579,6 +1586,7 @@ fsearch
 fuse
 fwupd
 fwupd-efi
+game-music-emu
 garcon
 gawk
 gbinder-python
@@ -1850,6 +1858,7 @@ irrlicht
 irrlichtmt
 irssi
 isa-l
+isl
 iso-codes
 isodate
 isomd5sum
@@ -2256,6 +2265,7 @@ libmbim
 libmediaart
 libmemcached
 libmicrohttpd
+libmikmod
 libmnl
 libmodbus
 libmodplug
@@ -2376,6 +2386,7 @@ libva
 libva-intel-driver
 libva-utils
 libvdpau
+libversion
 libvirt
 libvirt-dbus
 libvirt-glib
@@ -3747,7 +3758,6 @@ pypi-alabaster
 pypi-altair
 pypi-altgraph
 pypi-amqp
-pypi-aniso8601
 pypi-annotated_types
 pypi-ansi2html
 pypi-ansible_builder
@@ -3783,7 +3793,6 @@ pypi-atpublic
 pypi-attr
 pypi-attrdict
 pypi-attrs
-pypi-autocommand
 pypi-automat
 pypi-awesomeversion
 pypi-awscrt
@@ -3801,7 +3810,6 @@ pypi-babel
 pypi-backcall
 pypi-backoff
 pypi-backports.ssl_match_hostname
-pypi-bash_kernel
 pypi-bashlex
 pypi-bcrypt
 pypi-beartype
@@ -3818,6 +3826,8 @@ pypi-blessed
 pypi-blinker
 pypi-blivet
 pypi-blockdiag
+pypi-blosc
+pypi-blosc2
 pypi-bokeh
 pypi-boolean.py
 pypi-booleanoperations
@@ -3838,14 +3848,12 @@ pypi-cachy
 pypi-cairocffi
 pypi-cairosvg
 pypi-calver
-pypi-capturer
 pypi-cattrs
 pypi-cerberus
 pypi-certbot
 pypi-certbot_dns_google
 pypi-certifi
 pypi-cffi
-pypi-cffsubr
 pypi-cfgv
 pypi-chai
 pypi-chaospy
@@ -3855,7 +3863,6 @@ pypi-check_jsonschema
 pypi-check_manifest
 pypi-cheetah3
 pypi-cheroot
-pypi-cherrypy
 pypi-cibuildwheel
 pypi-ciscoisesdk
 pypi-ciso8601
@@ -3867,11 +3874,11 @@ pypi-click_log
 pypi-clikit
 pypi-cloudevents
 pypi-cloudflare
+pypi-clr_artifact
 pypi-cmarkgfm
 pypi-cmd2
 pypi-codecov
 pypi-colorama
-pypi-coloredlogs
 pypi-colorlog
 pypi-columnize
 pypi-comm
@@ -3892,6 +3899,7 @@ pypi-coveralls
 pypi-cppy
 pypi-crashtest
 pypi-crcmod
+pypi-crypt_r
 pypi-cryptography
 pypi-cryptography_vectors
 pypi-cs
@@ -3914,6 +3922,7 @@ pypi-decorator
 pypi-deepmerge
 pypi-deepspeed
 pypi-dep_logic
+pypi-dependency_groups
 pypi-deprecated
 pypi-deprecation
 pypi-devpi_client
@@ -4108,10 +4117,8 @@ pypi-isort
 pypi-itsdangerous
 pypi-janus
 pypi-jaraco.classes
-pypi-jaraco.collections
 pypi-jaraco.context
 pypi-jaraco.functools
-pypi-jaraco.text
 pypi-jarn.viewdoc
 pypi-javaproperties
 pypi-jedi
@@ -4124,7 +4131,6 @@ pypi-jinxed
 pypi-jmespath
 pypi-joblib
 pypi-josepy
-pypi-jsmin
 pypi-json5
 pypi-jsondiff
 pypi-jsonlines
@@ -4172,6 +4178,7 @@ pypi-lazy
 pypi-lazy_loader
 pypi-lazy_object_proxy
 pypi-ldap3
+pypi-legacy_cgi
 pypi-levenshtein
 pypi-libarchive_c
 pypi-libevdev
@@ -4246,12 +4253,14 @@ pypi-nbformat
 pypi-nbsphinx
 pypi-nbval
 pypi-ndg_httpsclient
+pypi-ndindex
 pypi-neat_python
 pypi-nest_asyncio
 pypi-netapp_lib
 pypi-netdisco
 pypi-netifaces
 pypi-networkx
+pypi-nftables
 pypi-nh3
 pypi-nibabel
 pypi-nltk
@@ -4354,6 +4363,7 @@ pypi-progress
 pypi-progressbar
 pypi-prometheus_client
 pypi-prompt_toolkit
+pypi-propcache
 pypi-proto_plus
 pypi-protobuf
 pypi-psautohint
@@ -4369,7 +4379,6 @@ pypi-py3dns
 pypi-py3nvml
 pypi-py_cpuinfo
 pypi-pyaml
-pypi-pyarrow
 pypi-pyasn1
 pypi-pyasn1_modules
 pypi-pybind11
@@ -4409,7 +4418,6 @@ pypi-pylint
 pypi-pymdown_extensions
 pypi-pynacl
 pypi-pynetbox
-pypi-pynsist
 pypi-pynvml
 pypi-pyomo
 pypi-pyopengl
@@ -4508,7 +4516,6 @@ pypi-regress
 pypi-reportlab
 pypi-repoze.lru
 pypi-requests
-pypi-requests_download
 pypi-requests_file
 pypi-requests_gssapi
 pypi-requests_kerberos
@@ -4524,7 +4531,6 @@ pypi-retrolab
 pypi-retry
 pypi-retry_decorator
 pypi-retryz
-pypi-retype
 pypi-rfc3339_validator
 pypi-rfc3986
 pypi-rfc3986_validator
@@ -4633,6 +4639,7 @@ pypi-subprocess_tee
 pypi-suds
 pypi-sure
 pypi-svg.path
+pypi-tables
 pypi-tabulate
 pypi-tap.py
 pypi-tblib
@@ -4667,7 +4674,6 @@ pypi-tornado_xstatic
 pypi-tox
 pypi-tqdm
 pypi-traitlets
-pypi-trampolim
 pypi-transformers
 pypi-translationstring
 pypi-trimesh
@@ -4677,7 +4683,6 @@ pypi-trove_classifiers
 pypi-truststore
 pypi-ttp
 pypi-twine
-pypi-typed_ast
 pypi-typeguard
 pypi-types_cffi
 pypi-types_colorama
@@ -4708,7 +4713,6 @@ pypi-tzdata
 pypi-tzlocal
 pypi-ua_parser
 pypi-uc_micro_py
-pypi-ufo2ft
 pypi-ufolib2
 pypi-uharfbuzz
 pypi-ujson
@@ -4989,6 +4993,7 @@ scour
 scowl
 screen
 scummvm
+scx
 sddm
 sddm-kcm
 seahorse
@@ -5169,7 +5174,6 @@ usb-modeswitch-data
 usbredir
 usbutils
 userspace-rcu
-usrbinjava
 usrbinvi
 usrsctp
 utf8proc
@@ -5315,6 +5319,7 @@ xrandr
 xrdb
 xrdp
 xrestop
+xscorch
 xscreensaver
 xsel
 xset

vendor.py

@@ -50,16 +50,17 @@ def setup_content(url):
 
 
 def setup_cargo_vendor(path):
+    cargo_paths = []
     for dirpath, _, files in os.walk(path):
         for fname in files:
             if fname == "Cargo.toml":
-                return dirpath
-    return False
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
 
 
-def update_cargo_vendor(path, name, git):
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
     git_uri = os.path.join(git, name)
-    vendor_path = os.path.join(path, 'vendor')
+    vendor_path = os.path.join(tmpdir, 'vendor')
     subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
                    stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
@@ -69,16 +70,18 @@ def update_cargo_vendor(path, name, git):
                        stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
                        shell=True, check=True, stdout=subprocess.DEVNULL)
-    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
-    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    cargo_vendors = subprocess.run('cargo vendor', cwd=path, shell=True,
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
                                    check=True, stdout=subprocess.PIPE,
                                    universal_newlines=True).stdout
     with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
         gafile.write("* text=false\n")
-    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
@@ -111,11 +114,11 @@ def update_cargo_sources(name, tag, cargo_vendors):
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_replace}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
                 print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
@@ -128,11 +131,11 @@ def update_cargo_sources(name, tag, cargo_vendors):
     with open('options.conf', encoding='utf8') as ofile:
         for line in ofile.readlines():
             if line.startswith('archives'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
                 options.append(new_archives)
             else:
                 options.append(line)
@@ -152,11 +155,11 @@ def main():
 
     tdir = setup_content(args.url)
     if vtype == 'cargo':
-        vdir = setup_cargo_vendor(tdir)
-        if not vdir:
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
             print(args.archives)
         else:
-            tag, cargo_vendors = update_cargo_vendor(vdir, args.name, args.git)
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
             update_cargo_sources(args.name, tag, cargo_vendors)
     shutil.rmtree(tdir)