Update packages file for version 38680

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile.common

@@ -157,6 +157,10 @@ pullrebase:
 		fi \
 	fi
 
+# help vendor: Attempt to create an updated vendor tar archive if needed
+vendor: preautospec-checks pullrebase clean-old-content
+	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
+
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -171,6 +175,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		exit 1; \
 	fi
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
+	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \

Makefile.config

@@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs
 # Location where project repos are hosted.
 PRJ_BASE_URL = https://github.com/clearlinux
 
+# Location where vendor repos are hosted.
+VND_BASE_URL =
+
 # Project repos that are integral to the common tooling.
 PROJ_PKGS = autospec clr-bundles common
 

packages

@@ -492,6 +492,7 @@ R-htmlwidgets
 R-httpcode
 R-httpuv
 R-httr
+R-httr2
 R-hunspell
 R-hwriter
 R-hypergeo
@@ -1034,6 +1035,7 @@ Vulkan-Headers
 Vulkan-Loader
 Vulkan-Tools
 WALinuxAgent
+WPEBackend-fdo
 WireGuard
 XStatic-term.js
 Z3
@@ -2292,6 +2294,7 @@ libwebp
 libwnck
 libwnck3
 libwpd
+libwpe
 libwpg
 libwps
 libxcvt
@@ -3624,6 +3627,7 @@ pyotp
 pyparted
 pypi-absl_py
 pypi-abydos
+pypi-accelerate
 pypi-acme
 pypi-adal
 pypi-aiodns
@@ -3666,6 +3670,7 @@ pypi-asyncssh
 pypi-asynctest
 pypi-atomicwrites
 pypi-atpublic
+pypi-attrdict
 pypi-attrs
 pypi-autocommand
 pypi-automat
@@ -3689,6 +3694,7 @@ pypi-bashlex
 pypi-bcrypt
 pypi-beautifulsoup4
 pypi-beniget
+pypi-billiard
 pypi-binaryornot
 pypi-bindep
 pypi-bitmath
@@ -3716,6 +3722,8 @@ pypi-cached_property
 pypi-cachetools
 pypi-cachez
 pypi-cachy
+pypi-cairocffi
+pypi-cairosvg
 pypi-calver
 pypi-capturer
 pypi-cattrs
@@ -3749,6 +3757,7 @@ pypi-cmarkgfm
 pypi-cmd2
 pypi-codecov
 pypi-colorama
+pypi-coloredlogs
 pypi-colorlog
 pypi-columnize
 pypi-comm
@@ -3787,11 +3796,13 @@ pypi-datetime
 pypi-debugpy
 pypi-decorator
 pypi-deepmerge
+pypi-deepspeed
 pypi-deprecated
 pypi-deprecation
 pypi-diff_cover
 pypi-diff_match_patch
 pypi-diffoscope
+pypi-diffusers
 pypi-dill
 pypi-diskcache
 pypi-distlib
@@ -3825,6 +3836,7 @@ pypi-eternalegypt
 pypi-etils
 pypi-etuples
 pypi-evdev
+pypi-eventlet
 pypi-exceptiongroup
 pypi-exdir
 pypi-execnet
@@ -3832,6 +3844,7 @@ pypi-executing
 pypi-expandvars
 pypi-expecttest
 pypi-fabric
+pypi-fairscale
 pypi-falcon
 pypi-fasteners
 pypi-fastimport
@@ -3846,6 +3859,7 @@ pypi-flake8_docstrings
 pypi-flake8_import_order
 pypi-flaky
 pypi-flask
+pypi-flatbuffers
 pypi-flatten_json
 pypi-flit
 pypi-flit_core
@@ -3865,6 +3879,7 @@ pypi-geventhttpclient
 pypi-ghp_import
 pypi-gitdb
 pypi-gitpython
+pypi-glad
 pypi-glfw
 pypi-gmqtt
 pypi-google_api_core
@@ -3905,6 +3920,7 @@ pypi-hatch_requirements_txt
 pypi-hatch_vcs
 pypi-hatchling
 pypi-heapdict
+pypi-hjson
 pypi-hkp4py
 pypi-home_assistant_frontend
 pypi-hpack
@@ -3914,6 +3930,7 @@ pypi-httpcore
 pypi-httplib2
 pypi-httpretty
 pypi-httpx
+pypi-huggingface_hub
 pypi-humanfriendly
 pypi-humanize
 pypi-hyperframe
@@ -3937,6 +3954,7 @@ pypi-inflect
 pypi-inflection
 pypi-influxdb
 pypi-iniconfig
+pypi-installer
 pypi-intervaltree
 pypi-invoke
 pypi-iocapture
@@ -3992,6 +4010,7 @@ pypi-jupyter_server_terminals
 pypi-jupyter_sphinx
 pypi-jupyterlab
 pypi-jupyterlab_launcher
+pypi-jupyterlab_link_share
 pypi-jupyterlab_pygments
 pypi-jupyterlab_server
 pypi-jupyterlab_widgets
@@ -4007,10 +4026,13 @@ pypi-kubernetes
 pypi-langcodes
 pypi-lark_parser
 pypi-latexcodec
+pypi-lavacli
+pypi-lazy_loader
 pypi-lazy_object_proxy
 pypi-ldap3
 pypi-libarchive_c
 pypi-libevdev
+pypi-lightning_utilities
 pypi-linkify_it_py
 pypi-livereload
 pypi-llfuse
@@ -4027,6 +4049,7 @@ pypi-markdown2
 pypi-markdown_it_py
 pypi-matplotlib
 pypi-matplotlib_inline
+pypi-maturin
 pypi-mdit_py_plugins
 pypi-mdurl
 pypi-mergedeep
@@ -4093,12 +4116,14 @@ pypi-onnx
 pypi-opcodes
 pypi-openshift
 pypi-opt_einsum
+pypi-ordered_set
 pypi-ouimeaux
 pypi-outcome
 pypi-ovirt_imageio
 pypi-packaging
 pypi-paho_mqtt
 pypi-pandocfilters
+pypi-parameterized
 pypi-paramiko
 pypi-parse
 pypi-parse_type
@@ -4128,6 +4153,7 @@ pypi-phabricator
 pypi-pickleshare
 pypi-picobox
 pypi-pid
+pypi-pidigits
 pypi-pillow
 pypi-pint
 pypi-pip
@@ -4165,6 +4191,7 @@ pypi-pure_eval
 pypi-purestorage
 pypi-purity_fb
 pypi-py
+pypi-py3nvml
 pypi-py_cpuinfo
 pypi-pyaml
 pypi-pyasn1
@@ -4273,6 +4300,7 @@ pypi-python_version
 pypi-python_xlib
 pypi-pythran
 pypi-pytoolconfig
+pypi-pytorch_lightning
 pypi-pytz
 pypi-pytz_deprecation_shim
 pypi-pyu2f
@@ -4309,6 +4337,7 @@ pypi-requirements_parser
 pypi-resolvelib
 pypi-responses
 pypi-restructuredtext_lint
+pypi-retrolab
 pypi-retry
 pypi-retry_decorator
 pypi-retryz
@@ -4344,6 +4373,7 @@ pypi-selenium
 pypi-selinux
 pypi-semantic_version
 pypi-send2trash
+pypi-sentencepiece
 pypi-sentinels
 pypi-sentry_sdk
 pypi-seqdiag
@@ -4360,6 +4390,7 @@ pypi-shapely
 pypi-shellingham
 pypi-simplegeneric
 pypi-six
+pypi-slugify
 pypi-smart_open
 pypi-smartypants
 pypi-smmap
@@ -4412,6 +4443,7 @@ pypi-stack_data
 pypi-statsmodels
 pypi-storops
 pypi-subprocess_tee
+pypi-suds
 pypi-sure
 pypi-svg.path
 pypi-tabulate
@@ -4433,11 +4465,13 @@ pypi-three_merge
 pypi-tifffile
 pypi-tinycss2
 pypi-tldextract
+pypi-tokenizers
 pypi-toml
 pypi-tomli
 pypi-tomli_w
 pypi-tomlkit
 pypi-toolz
+pypi-torchmetrics
 pypi-tornado
 pypi-tornado_xstatic
 pypi-tox
@@ -4491,6 +4525,7 @@ pypi-urwid
 pypi-user_agents
 pypi-userpath
 pypi-uvloop
+pypi-verboselogs
 pypi-vine
 pypi-virtualenv
 pypi-volatile
@@ -4516,6 +4551,7 @@ pypi-wrapt
 pypi-wsgiproxy2
 pypi-wsproto
 pypi-wurlitzer
+pypi-xgboost
 pypi-xmldiff
 pypi-xmljson
 pypi-xmlschema
@@ -4535,11 +4571,13 @@ pypi-xstatic_objectpath
 pypi-xstatic_roboto_fontface
 pypi-xvfbwrapper
 pypi-xyzservices
+pypi-y_py
 pypi-yamllint
 pypi-yamlloader
 pypi-yapf
 pypi-yarg
 pypi-yarl
+pypi-yattag
 pypi-zabbix_api
 pypi-zc.lockfile
 pypi-zeroconf
@@ -4663,6 +4701,7 @@ rest
 restic
 rhythmbox
 rinutils
+ripgrep
 rlwrap
 rng-tools
 robin-map
@@ -4707,6 +4746,7 @@ seahorse
 seatd
 sed
 seer
+sentencepiece
 serf
 setserial
 setxkbmap
@@ -4792,7 +4832,6 @@ sympy
 syndication
 synergy
 syntax-highlighting
-sysdig
 syslinux
 sysprof
 sysstat
@@ -4908,6 +4947,7 @@ wayland-protocols
 wcslib
 webkitgtk
 webkitgtk50
+webkitgtk60
 weechat
 weston
 wget

vendor.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+from git import Repo
+
+import requests
+
+
+def vendor_check():
+    if not os.path.isfile('options.conf'):
+        return False
+    config = configparser.ConfigParser(interpolation=None)
+    config.read('options.conf')
+    if 'autospec' not in config.sections():
+        return False
+    if vendor := config['autospec'].get('cargo_vendor'):
+        if vendor == "true":
+            return 'cargo'
+
+    return False
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('url')
+    parser.add_argument('name')
+    parser.add_argument('git')
+    return parser.parse_args()
+
+
+def setup_content(url):
+    tdir = tempfile.mkdtemp()
+    outfile = os.path.join(tdir, os.path.basename(url))
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()
+    with open(outfile, 'wb') as cfile:
+        cfile.write(response.content)
+
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    os.remove(outfile)
+    return tdir
+
+
+def setup_cargo_vendor(path):
+    for dirpath, _, files in os.walk(path):
+        for fname in files:
+            if fname == "Cargo.toml":
+                return dirpath
+    return False
+
+
+def update_cargo_vendor(path, name, git):
+    git_uri = os.path.join(git, name)
+    vendor_path = os.path.join(path, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    vendor_git = os.path.join(vendor_path, '.git')
+    if not os.path.isdir(vendor_git):
+        # initialize a git repo
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
+                       shell=True, check=True)
+    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+                   shell=True, check=True)
+    shutil.rmtree(vendor_path)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+                   shell=True, check=True)
+    repo = Repo(vendor_path)
+    if not len(repo.untracked_files) > 0 or repo.is_dirty():
+        return False
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
+                   shell=True, check=True)
+    gmt = time.gmtime()
+    tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
+    subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
+                   check=True)
+    subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
+                   shell=True, check=True)
+    time.sleep(30)
+    return tag
+
+
+def update_cargo_sources(name, tag):
+    makefile = []
+    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                 'snapshot', name)
+    with open('Makefile', encoding='utf8') as mfile:
+        for line in mfile.readlines():
+            if line.startswith('ARCHIVES'):
+                if re.match(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                else:
+                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                makefile.append(new_archives)
+            else:
+                makefile.append(line)
+    with open('Makefile', 'w', encoding='utf8') as mfile:
+        mfile.writelines(makefile)
+
+
+def main():
+    vtype = vendor_check()
+    if not vtype:
+        return
+
+    args = get_args()
+    tdir = setup_content(args.url)
+    if vtype == 'cargo':
+        vdir = setup_cargo_vendor(tdir)
+        if vdir:
+            tag = update_cargo_vendor(vdir, args.name, args.git)
+            if tag:
+                update_cargo_sources(args.name, tag)
+    shutil.rmtree(tdir)
+
+if __name__ == '__main__':
+    main()