update packages file

Some projects store multiple Cargo.toml files, let vendor handle all
of them (might be worth having some config for certain Cargo.toml
paths to skip in the future though).

Signed-off-by: William Douglas <william.douglas@intel.com>

packages

@@ -1117,6 +1117,7 @@ ansible
 ansible-core
 antlr4-python3-runtime
 apache-ant
+apache-arrow
 appstream
 appstream-glib
 apr
@@ -1237,6 +1238,7 @@ byobu
 bz2file
 bzip2
 c-ares
+c-blosc
 c-blosc2
 cJSON
 c_rehash
@@ -1331,17 +1333,20 @@ colord-kde
 colordiff
 columbiad
 compat-Botan-soname2
+compat-abseil-cpp-rolling
 compat-atkmm-soname16
 compat-babeltrace-one
 compat-cairomm-soname10
 compat-codec2-soname1
 compat-enchant-soname1
 compat-ffmpeg-4.4
+compat-ffmpeg-6
 compat-fuse-soname2
 compat-gcc-10
 compat-gcr-soname1
 compat-glibmm-soname24
 compat-gnome-bluetooth-soname-13
+compat-grpc-soname66
 compat-gsl-soname27
 compat-gtksourceview-soname3
 compat-icu4c-rolling
@@ -1357,6 +1362,7 @@ compat-libvpx-soname8
 compat-pangomm-soname14
 compat-protobuf-soname29
 compat-protobuf-soname32
+compat-python3-rolling
 compat-re2-soname10
 compat-readline-soname5
 compat-taglib-soname1
@@ -1398,6 +1404,7 @@ cups-filters
 cups-pk-helper
 curl
 cycler
+cyme
 cyrus-sasl
 dapl
 darktable
@@ -2339,6 +2346,7 @@ libsoup
 libspatialindex
 libspatialite
 libspectre
+libspelling
 libspiro
 libspnav
 libspng
@@ -2433,6 +2441,7 @@ llvm14
 llvm15
 llvm16
 llvm17
+llvm18
 lm-sensors
 lmdb
 logrotate
@@ -2595,6 +2604,7 @@ nettle
 network-manager-applet
 networkmanager-qt
 newt
+nextcloud-desktop
 nfs-utils
 nftables
 nghttp2
@@ -3741,7 +3751,6 @@ pypi-alabaster
 pypi-altair
 pypi-altgraph
 pypi-amqp
-pypi-aniso8601
 pypi-annotated_types
 pypi-ansi2html
 pypi-ansible_builder
@@ -3777,7 +3786,6 @@ pypi-atpublic
 pypi-attr
 pypi-attrdict
 pypi-attrs
-pypi-autocommand
 pypi-automat
 pypi-awesomeversion
 pypi-awscrt
@@ -3795,7 +3803,6 @@ pypi-babel
 pypi-backcall
 pypi-backoff
 pypi-backports.ssl_match_hostname
-pypi-bash_kernel
 pypi-bashlex
 pypi-bcrypt
 pypi-beartype
@@ -3812,6 +3819,8 @@ pypi-blessed
 pypi-blinker
 pypi-blivet
 pypi-blockdiag
+pypi-blosc
+pypi-blosc2
 pypi-bokeh
 pypi-boolean.py
 pypi-booleanoperations
@@ -3832,14 +3841,12 @@ pypi-cachy
 pypi-cairocffi
 pypi-cairosvg
 pypi-calver
-pypi-capturer
 pypi-cattrs
 pypi-cerberus
 pypi-certbot
 pypi-certbot_dns_google
 pypi-certifi
 pypi-cffi
-pypi-cffsubr
 pypi-cfgv
 pypi-chai
 pypi-chaospy
@@ -3849,7 +3856,6 @@ pypi-check_jsonschema
 pypi-check_manifest
 pypi-cheetah3
 pypi-cheroot
-pypi-cherrypy
 pypi-cibuildwheel
 pypi-ciscoisesdk
 pypi-ciso8601
@@ -3861,11 +3867,11 @@ pypi-click_log
 pypi-clikit
 pypi-cloudevents
 pypi-cloudflare
+pypi-clr_artifact
 pypi-cmarkgfm
 pypi-cmd2
 pypi-codecov
 pypi-colorama
-pypi-coloredlogs
 pypi-colorlog
 pypi-columnize
 pypi-comm
@@ -3886,6 +3892,7 @@ pypi-coveralls
 pypi-cppy
 pypi-crashtest
 pypi-crcmod
+pypi-crypt_r
 pypi-cryptography
 pypi-cryptography_vectors
 pypi-cs
@@ -3943,8 +3950,11 @@ pypi-doxypypy
 pypi-doxyqml
 pypi-dpcontracts
 pypi-dulwich
+pypi-durationpy
 pypi-ecdsa
 pypi-editables
+pypi-editdistance
+pypi-edlib
 pypi-einops
 pypi-elementpath
 pypi-enrich
@@ -4099,10 +4109,8 @@ pypi-isort
 pypi-itsdangerous
 pypi-janus
 pypi-jaraco.classes
-pypi-jaraco.collections
 pypi-jaraco.context
 pypi-jaraco.functools
-pypi-jaraco.text
 pypi-jarn.viewdoc
 pypi-javaproperties
 pypi-jedi
@@ -4115,7 +4123,6 @@ pypi-jinxed
 pypi-jmespath
 pypi-joblib
 pypi-josepy
-pypi-jsmin
 pypi-json5
 pypi-jsondiff
 pypi-jsonlines
@@ -4163,6 +4170,7 @@ pypi-lazy
 pypi-lazy_loader
 pypi-lazy_object_proxy
 pypi-ldap3
+pypi-legacy_cgi
 pypi-levenshtein
 pypi-libarchive_c
 pypi-libevdev
@@ -4237,12 +4245,14 @@ pypi-nbformat
 pypi-nbsphinx
 pypi-nbval
 pypi-ndg_httpsclient
+pypi-ndindex
 pypi-neat_python
 pypi-nest_asyncio
 pypi-netapp_lib
 pypi-netdisco
 pypi-netifaces
 pypi-networkx
+pypi-nftables
 pypi-nh3
 pypi-nibabel
 pypi-nltk
@@ -4331,6 +4341,7 @@ pypi-poetry
 pypi-poetry_core
 pypi-poetry_plugin_export
 pypi-polib
+pypi-polyleven
 pypi-port_for
 pypi-portalocker
 pypi-portend
@@ -4344,6 +4355,7 @@ pypi-progress
 pypi-progressbar
 pypi-prometheus_client
 pypi-prompt_toolkit
+pypi-propcache
 pypi-proto_plus
 pypi-protobuf
 pypi-psautohint
@@ -4372,6 +4384,7 @@ pypi-pycollada
 pypi-pycosat
 pypi-pycountry
 pypi-pycparser
+pypi-pycrdt
 pypi-pycryptodome
 pypi-pycryptodomex
 pypi-pycups
@@ -4397,7 +4410,6 @@ pypi-pylint
 pypi-pymdown_extensions
 pypi-pynacl
 pypi-pynetbox
-pypi-pynsist
 pypi-pynvml
 pypi-pyomo
 pypi-pyopengl
@@ -4496,7 +4508,6 @@ pypi-regress
 pypi-reportlab
 pypi-repoze.lru
 pypi-requests
-pypi-requests_download
 pypi-requests_file
 pypi-requests_gssapi
 pypi-requests_kerberos
@@ -4512,7 +4523,6 @@ pypi-retrolab
 pypi-retry
 pypi-retry_decorator
 pypi-retryz
-pypi-retype
 pypi-rfc3339_validator
 pypi-rfc3986
 pypi-rfc3986_validator
@@ -4621,6 +4631,7 @@ pypi-subprocess_tee
 pypi-suds
 pypi-sure
 pypi-svg.path
+pypi-tables
 pypi-tabulate
 pypi-tap.py
 pypi-tblib
@@ -4635,6 +4646,7 @@ pypi-text_unidecode
 pypi-textdistance
 pypi-textfsm
 pypi-texttable
+pypi-thefuzz
 pypi-threadpoolctl
 pypi-three_merge
 pypi-tifffile
@@ -4654,7 +4666,6 @@ pypi-tornado_xstatic
 pypi-tox
 pypi-tqdm
 pypi-traitlets
-pypi-trampolim
 pypi-transformers
 pypi-translationstring
 pypi-trimesh
@@ -4664,7 +4675,6 @@ pypi-trove_classifiers
 pypi-truststore
 pypi-ttp
 pypi-twine
-pypi-typed_ast
 pypi-typeguard
 pypi-types_cffi
 pypi-types_colorama
@@ -4695,7 +4705,6 @@ pypi-tzdata
 pypi-tzlocal
 pypi-ua_parser
 pypi-uc_micro_py
-pypi-ufo2ft
 pypi-ufolib2
 pypi-uharfbuzz
 pypi-ujson

vendor.py

@@ -50,16 +50,17 @@ def setup_content(url):
 
 
 def setup_cargo_vendor(path):
+    cargo_paths = []
     for dirpath, _, files in os.walk(path):
         for fname in files:
             if fname == "Cargo.toml":
-                return dirpath
-    return False
+                cargo_paths.append(os.path.join(dirpath, fname))
+    return cargo_paths
 
 
-def update_cargo_vendor(path, name, git):
+def update_cargo_vendor(tmpdir, cargo_paths, name, git):
     git_uri = os.path.join(git, name)
-    vendor_path = os.path.join(path, 'vendor')
+    vendor_path = os.path.join(tmpdir, 'vendor')
     subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
                    stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
@@ -69,16 +70,18 @@ def update_cargo_vendor(path, name, git):
                        stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
                        shell=True, check=True, stdout=subprocess.DEVNULL)
-    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
-    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+    backup_vendor_git = os.path.join(tmpdir, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    cargo_vendors = subprocess.run('cargo vendor', cwd=path, shell=True,
+    vendor_cmd = 'cargo vendor ' + ' '.join([f"-s {x}" for x in cargo_paths[:-1]])
+    vendor_cmd += f" --manifest-path {cargo_paths[-1]}"
+    cargo_vendors = subprocess.run(vendor_cmd, cwd=tmpdir, shell=True,
                                    check=True, stdout=subprocess.PIPE,
                                    universal_newlines=True).stdout
     with open(os.path.join(vendor_path, ".gitattributes"), "w", encoding='utf8') as gafile:
         gafile.write("* text=false\n")
-    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=tmpdir,
                    shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
@@ -111,11 +114,11 @@ def update_cargo_sources(name, tag, cargo_vendors):
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_replace}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_replace}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.gz ./vendor\n"
                 print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
@@ -128,11 +131,11 @@ def update_cargo_sources(name, tag, cargo_vendors):
     with open('options.conf', encoding='utf8') as ofile:
         for line in ofile.readlines():
             if line.startswith('archives'):
-                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.gz',
+                                          f"{archive_match}-{tag}.tar.gz", line)
                 else:
-                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.gz ./vendor\n"
                 options.append(new_archives)
             else:
                 options.append(line)
@@ -152,11 +155,11 @@ def main():
 
     tdir = setup_content(args.url)
     if vtype == 'cargo':
-        vdir = setup_cargo_vendor(tdir)
-        if not vdir:
+        cargo_paths = setup_cargo_vendor(tdir)
+        if len(cargo_paths) == 0:
             print(args.archives)
         else:
-            tag, cargo_vendors = update_cargo_vendor(vdir, args.name, args.git)
+            tag, cargo_vendors = update_cargo_vendor(tdir, cargo_paths, args.name, args.git)
             update_cargo_sources(args.name, tag, cargo_vendors)
     shutil.rmtree(tdir)