Add support for multiple Cargo.toml files

Some projects store multiple Cargo.toml files, let vendor handle all
of them (might be worth having some config for certain Cargo.toml
paths to skip in the future though).

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -223,6 +223,7 @@ scanlicense:
 #help the variable BUMP_MSG is set, its value is used as the commit summary.
 #help Otherwise a generic commit summary is used.
 bump:
+	git stash
 	git pull --rebase
 	$(MAKE) bumpnogit
 	git add $(SPECFILE) release

packages

@@ -1117,6 +1117,7 @@ ansible
 ansible-core
 antlr4-python3-runtime
 apache-ant
+apache-arrow
 appstream
 appstream-glib
 apr
@@ -1237,6 +1238,7 @@ byobu
 bz2file
 bzip2
 c-ares
+c-blosc
 c-blosc2
 cJSON
 c_rehash
@@ -1331,6 +1333,7 @@ colord-kde
 colordiff
 columbiad
 compat-Botan-soname2
+compat-abseil-cpp-rolling
 compat-atkmm-soname16
 compat-babeltrace-one
 compat-cairomm-soname10
@@ -1342,6 +1345,7 @@ compat-gcc-10
 compat-gcr-soname1
 compat-glibmm-soname24
 compat-gnome-bluetooth-soname-13
+compat-grpc-soname66
 compat-gsl-soname27
 compat-gtksourceview-soname3
 compat-icu4c-rolling
@@ -1398,6 +1402,7 @@ cups-filters
 cups-pk-helper
 curl
 cycler
+cyme
 cyrus-sasl
 dapl
 darktable
@@ -2339,6 +2344,7 @@ libsoup
 libspatialindex
 libspatialite
 libspectre
+libspelling
 libspiro
 libspnav
 libspng
@@ -2433,6 +2439,7 @@ llvm14
 llvm15
 llvm16
 llvm17
+llvm18
 lm-sensors
 lmdb
 logrotate
@@ -2595,6 +2602,7 @@ nettle
 network-manager-applet
 networkmanager-qt
 newt
+nextcloud-desktop
 nfs-utils
 nftables
 nghttp2
@@ -3812,6 +3820,8 @@ pypi-blessed
 pypi-blinker
 pypi-blivet
 pypi-blockdiag
+pypi-blosc
+pypi-blosc2
 pypi-bokeh
 pypi-boolean.py
 pypi-booleanoperations
@@ -3943,8 +3953,11 @@ pypi-doxypypy
 pypi-doxyqml
 pypi-dpcontracts
 pypi-dulwich
+pypi-durationpy
 pypi-ecdsa
 pypi-editables
+pypi-editdistance
+pypi-edlib
 pypi-einops
 pypi-elementpath
 pypi-enrich
@@ -4237,12 +4250,14 @@ pypi-nbformat
 pypi-nbsphinx
 pypi-nbval
 pypi-ndg_httpsclient
+pypi-ndindex
 pypi-neat_python
 pypi-nest_asyncio
 pypi-netapp_lib
 pypi-netdisco
 pypi-netifaces
 pypi-networkx
+pypi-nftables
 pypi-nh3
 pypi-nibabel
 pypi-nltk
@@ -4331,6 +4346,7 @@ pypi-poetry
 pypi-poetry_core
 pypi-poetry_plugin_export
 pypi-polib
+pypi-polyleven
 pypi-port_for
 pypi-portalocker
 pypi-portend
@@ -4359,6 +4375,7 @@ pypi-py3dns
 pypi-py3nvml
 pypi-py_cpuinfo
 pypi-pyaml
+pypi-pyarrow
 pypi-pyasn1
 pypi-pyasn1_modules
 pypi-pybind11
@@ -4372,6 +4389,7 @@ pypi-pycollada
 pypi-pycosat
 pypi-pycountry
 pypi-pycparser
+pypi-pycrdt
 pypi-pycryptodome
 pypi-pycryptodomex
 pypi-pycups
@@ -4621,6 +4639,7 @@ pypi-subprocess_tee
 pypi-suds
 pypi-sure
 pypi-svg.path
+pypi-tables
 pypi-tabulate
 pypi-tap.py
 pypi-tblib
@@ -4635,6 +4654,7 @@ pypi-text_unidecode
 pypi-textdistance
 pypi-textfsm
 pypi-texttable
+pypi-thefuzz
 pypi-threadpoolctl
 pypi-three_merge
 pypi-tifffile

vendor.py

@@ -50,16 +50,17 @@ def setup_content(url):
 
 
 def setup_cargo_vendor(path):
+    cargo_paths = []
     for dirpath, _, files in os.walk(path):
         for fname in files:
             if fname == "Cargo.toml":
-                return dirpath
-    return False
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
 
 
-def update_cargo_vendor(path, name, git):
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
     git_uri = os.path.join(git, name)
-    vendor_path = os.path.join(path, 'vendor')
+    vendor_path = os.path.join(tmpdir, 'vendor')
     subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
                    stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
@@ -69,16 +70,18 @@ def update_cargo_vendor(path, name, git):
                        stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
                        shell=True, check=True, stdout=subprocess.DEVNULL)
-    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
-    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    cargo_vendors = subprocess.run('cargo vendor', cwd=path, shell=True,
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
                                    check=True, stdout=subprocess.PIPE,
                                    universal_newlines=True).stdout
     with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
         gafile.write("* text=false\n")
-    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
@@ -111,11 +114,11 @@ def update_cargo_sources(name, tag, cargo_vendors):
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_replace}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
                 print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
@@ -128,11 +131,11 @@ def update_cargo_sources(name, tag, cargo_vendors):
     with open('options.conf', encoding='utf8') as ofile:
         for line in ofile.readlines():
             if line.startswith('archives'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
                 options.append(new_archives)
             else:
                 options.append(line)
@@ -152,11 +155,11 @@ def main():
 
     tdir = setup_content(args.url)
     if vtype == 'cargo':
-        vdir = setup_cargo_vendor(tdir)
-        if not vdir:
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
             print(args.archives)
         else:
-            tag, cargo_vendors = update_cargo_vendor(vdir, args.name, args.git)
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
             update_cargo_sources(args.name, tag, cargo_vendors)
     shutil.rmtree(tdir)