update packages

Changes vendor from a target (it could still have a stand-alone target
but it wouldn't be used by the autospec target).

This change is needed because the Makefile is rewritten in the
autospec target but the ARCHIVES make variable is going to use the
value when make is executed. The common change causes the ARCHIVES
value to be eval'd during the autospec target execution in order for
the update from the vendor script to be seen by make.

The vendor script changes fix bugs in updating the options.conf and
Makefile. It also now handles returning the original ARCHIVES value as
well as prevent subshell scripts from outputting and adding badness
into the eval'd ARCHIVES variable.

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -157,10 +157,6 @@ pullrebase:
 		fi \
 	fi
 
-# help vendor: Attempt to create an updated vendor tar archive if needed
-vendor: preautospec-checks pullrebase clean-old-content
-	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
-
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -174,8 +170,8 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
-	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \

packages

@@ -48,6 +48,7 @@ PySocks
 PyYAML
 QAT-ZSTD-Plugin
 QAT_engine
+QR-Code-generator
 QXlsx
 R
 R-AER
@@ -110,6 +111,7 @@ R-ISOcodes
 R-ISwR
 R-Iso
 R-JM
+R-KMsurv
 R-Lahman
 R-LearnBayes
 R-Luminescence
@@ -376,6 +378,7 @@ R-estimability
 R-etm
 R-evaluate
 R-evd
+R-exactRankTests
 R-expint
 R-expm
 R-expsmooth
@@ -414,6 +417,7 @@ R-fixest
 R-flashClust
 R-flexmix
 R-flexsurv
+R-flexsurvcure
 R-flextable
 R-float
 R-fontBitstreamVera
@@ -461,9 +465,13 @@ R-ggfortify
 R-ggjoy
 R-ggplot2
 R-ggplot2movies
+R-ggpubr
 R-ggrepel
 R-ggridges
+R-ggsci
+R-ggsignif
 R-ggstats
+R-ggtext
 R-ggvis
 R-gh
 R-git2r
@@ -483,6 +491,7 @@ R-gower
 R-gplots
 R-gridBase
 R-gridExtra
+R-gridtext
 R-gsl
 R-gss
 R-gtable
@@ -537,6 +546,7 @@ R-kernlab
 R-kimisc
 R-kit
 R-klaR
+R-km.ci
 R-kmi
 R-knitr
 R-ks
@@ -597,6 +607,7 @@ R-mathjaxr
 R-matrixStats
 R-matrixcalc
 R-maxLik
+R-maxstat
 R-mclogit
 R-mclust
 R-mcmc
@@ -613,6 +624,7 @@ R-mime
 R-miniUI
 R-minpack.lm
 R-minqa
+R-minty
 R-mirt
 R-misc3d
 R-miscF
@@ -843,6 +855,7 @@ R-rsdmx
 R-rstan
 R-rstanarm
 R-rstantools
+R-rstatix
 R-rstpm2
 R-rstudioapi
 R-rsvg
@@ -923,7 +936,9 @@ R-styler
 R-subplex
 R-subselect
 R-superpc
+R-survMisc
 R-survey
+R-survminer
 R-svUnit
 R-svglite
 R-sys
@@ -1066,7 +1081,6 @@ acl
 acpica-unix2
 ade
 adwaita-icon-theme
-aesara
 akonadi
 akonadi-calendar
 akonadi-calendar-tools
@@ -1175,6 +1189,7 @@ boto3
 botocore
 bovo
 box2d
+bpftool
 bpftrace
 breeze
 breeze-gtk
@@ -1223,6 +1238,7 @@ capslock
 capstone
 casync
 catch2
+cbindgen
 ccache
 ccid
 cdparanoia
@@ -1309,6 +1325,7 @@ compat-gcc-10
 compat-gcr-soname1
 compat-glibmm-soname24
 compat-gnome-bluetooth-soname-13
+compat-gsl-soname27
 compat-gtksourceview-soname3
 compat-json-c-soname4
 compat-libffi-soname6
@@ -1321,6 +1338,7 @@ compat-libvpx-soname7
 compat-libvpx-soname8
 compat-pangomm-soname14
 compat-protobuf-soname29
+compat-protobuf-soname32
 compat-re2-soname10
 compat-readline-soname5
 compat-taglib-soname1
@@ -1544,7 +1562,6 @@ gc
 gcab
 gcc
 gcc11
-gcc14
 gcc7
 gcc8
 gcc9
@@ -1976,7 +1993,6 @@ koji
 kolf
 kollision
 kolourpaint
-kompare
 konqueror
 konquest
 konsole
@@ -2078,6 +2094,7 @@ libaccounts-glib
 libaccounts-qt
 libadwaita
 libaio
+libajantv2
 libao
 libaom
 libappindicator
@@ -2116,6 +2133,7 @@ libcryptui
 libcuckoo
 libcxx
 libdaemon
+libdatachannel
 libdatrie
 libdazzle
 libdbusmenu
@@ -2185,6 +2203,7 @@ libiscsi
 libisofs
 libjcat
 libjpeg-turbo
+libjuice
 libkcapi
 libkcddb
 libkcompactdisc
@@ -2561,8 +2580,10 @@ nicstat
 nim
 ninja
 nload
+nlohmann_json
 nlopt
 nmap
+node-addon-api
 nodejs
 nose-parameterized
 not-ffmpeg
@@ -2588,6 +2609,7 @@ nut
 nv-codec-headers
 nvme-cli
 oath-toolkit
+obs-studio
 ocaml
 ocamlbuild
 ocl-icd
@@ -2621,6 +2643,7 @@ openpace
 opensm
 openssh
 openssl
+openvino
 openvpn
 openvswitch
 optipng
@@ -3607,6 +3630,7 @@ plasma-welcome
 plasma-workspace
 plasma-workspace-wallpapers
 plasma5support
+plog
 plzip
 pm-graph
 pmdk
@@ -3917,6 +3941,7 @@ pypi-flaky
 pypi-flask
 pypi-flatbuffers
 pypi-flatten_json
+pypi-flexparser
 pypi-flit
 pypi-flit_core
 pypi-flit_scm
@@ -4038,7 +4063,6 @@ pypi-jaraco.functools
 pypi-jaraco.text
 pypi-jarn.viewdoc
 pypi-javaproperties
-pypi-jax
 pypi-jedi
 pypi-jeepney
 pypi-jellyfish
@@ -4086,6 +4110,7 @@ pypi-kiwisolver
 pypi-krb5
 pypi-kubernetes
 pypi-langcodes
+pypi-language_data
 pypi-lark_parser
 pypi-latexcodec
 pypi-lavacli
@@ -4111,6 +4136,7 @@ pypi-lxml
 pypi-lz4
 pypi-macholib
 pypi-mako
+pypi-marisa_trie
 pypi-markdown
 pypi-markdown2
 pypi-markdown_exec
@@ -4130,6 +4156,7 @@ pypi-mkdocs
 pypi-mkdocs_autorefs
 pypi-mkdocs_exclude
 pypi-mkdocs_gen_files
+pypi-mkdocs_get_deps
 pypi-mkdocs_htmlproofer_plugin
 pypi-mkdocs_material_extensions
 pypi-mkdocs_monorepo_plugin
@@ -4195,6 +4222,7 @@ pypi-outcome
 pypi-overrides
 pypi-ovirt_imageio
 pypi-packaging
+pypi-packaging_legacy
 pypi-paginate
 pypi-paho_mqtt
 pypi-pandocfilters
@@ -4260,6 +4288,7 @@ pypi-portend
 pypi-posix_ipc
 pypi-poyo
 pypi-pre_commit
+pypi-pretty_errors
 pypi-prettytable
 pypi-probed
 pypi-progress
@@ -4267,6 +4296,7 @@ pypi-progressbar
 pypi-prometheus_client
 pypi-prompt_toolkit
 pypi-proto_plus
+pypi-protobuf
 pypi-psautohint
 pypi-psutil
 pypi-psycopg2
@@ -4455,6 +4485,7 @@ pypi-safetensors
 pypi-salib
 pypi-sanic_routing
 pypi-scikit_build
+pypi-scikit_build_core
 pypi-scikit_image
 pypi-scikit_learn
 pypi-scp
@@ -4584,6 +4615,7 @@ pypi-ttp
 pypi-twine
 pypi-typed_ast
 pypi-typeguard
+pypi-types_cffi
 pypi-types_colorama
 pypi-types_cryptography
 pypi-types_dataclasses
@@ -4641,6 +4673,7 @@ pypi-vsts_cd_manager
 pypi-waitress
 pypi-warlock
 pypi-watchdog
+pypi-watchfiles
 pypi-wbdata
 pypi-wcmatch
 pypi-wcwidth
@@ -4738,6 +4771,7 @@ qca
 qca-qt5
 qcoro
 qcoro6
+qdmr
 qemu
 qemu-guest-additions
 qgit
@@ -4869,6 +4903,7 @@ rtmpdump
 rttr
 ruby
 runc
+rust-bindgen
 rustc
 rxvt-unicode
 s2tc
@@ -5068,8 +5103,10 @@ usbutils
 userspace-rcu
 usrbinjava
 usrbinvi
+usrsctp
 utf8proc
 utfcpp
+uthash
 util-linux
 util-macros
 uwsgi
@@ -5176,6 +5213,7 @@ xfce4-whiskermenu-plugin
 xfconf
 xfdesktop
 xfontsel
+xfsdump
 xfsprogs
 xfwm4
 xhost

vendor.py

@@ -30,6 +30,7 @@ def vendor_check():
 
 def get_args():
     parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
     parser.add_argument('url')
     parser.add_argument('name')
     parser.add_argument('git')
@@ -44,7 +45,7 @@ def setup_content(url):
     with open(outfile, 'wb') as cfile:
         cfile.write(response.content)
 
-    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
     os.remove(outfile)
     return tdir
 
@@ -60,32 +61,36 @@ def setup_cargo_vendor(path):
 def update_cargo_vendor(path, name, git):
     git_uri = os.path.join(git, name)
     vendor_path = os.path.join(path, 'vendor')
-    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
     if not os.path.isdir(vendor_git):
         # initialize a git repo
-        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
-                       shell=True, check=True)
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
     backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
     subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
         return False
-    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     gmt = time.gmtime()
     tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
     subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
-                   check=True)
+                   check=True, stdout=subprocess.DEVNULL)
     subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     time.sleep(30)
     return tag
 
@@ -93,30 +98,35 @@ def update_cargo_vendor(path, name, git):
 def update_cargo_sources(name, tag):
     makefile = []
     options = []
-    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
                                  'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.match(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
                     new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                                          f"{archive_replace}-{tag}.tar.xz", line)
                 else:
-                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
                 makefile.append(line)
     with open('Makefile', 'w', encoding='utf8') as mfile:
         mfile.writelines(makefile)
 
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
     with open('options.conf', encoding='utf8') as ofile:
         for line in ofile.readlines():
             if line.startswith('archives'):
-                if re.match(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
                     new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                                          f"{archive_match}-{tag}.tar.xz", line)
                 else:
-                    new_archives = f"achives = {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
                 options.append(new_archives)
             else:
                 options.append(line)
@@ -125,11 +135,14 @@ def update_cargo_sources(name, tag):
 
 
 def main():
+    updated = False
+    args = get_args()
+
     vtype = vendor_check()
     if not vtype:
+        print(args.archives)
         return
 
-    args = get_args()
     tdir = setup_content(args.url)
     if vtype == 'cargo':
         vdir = setup_cargo_vendor(tdir)
@@ -137,7 +150,11 @@ def main():
             tag = update_cargo_vendor(vdir, args.name, args.git)
             if tag:
                 update_cargo_sources(args.name, tag)
+                updated = True
+    if not updated:
+        print(args.archives)
     shutil.rmtree(tdir)
 
+
 if __name__ == '__main__':
     main()