Update packages file for version 38710

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile.common

@@ -157,6 +157,10 @@ pullrebase:
 		fi \
 	fi
 
+# help vendor: Attempt to create an updated vendor tar archive if needed
+vendor: preautospec-checks pullrebase clean-old-content
+	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
+
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -171,6 +175,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		exit 1; \
 	fi
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
+	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \

Makefile.config

@@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs
 # Location where project repos are hosted.
 PRJ_BASE_URL = https://github.com/clearlinux
 
+# Location where vendor repos are hosted.
+VND_BASE_URL =
+
 # Project repos that are integral to the common tooling.
 PROJ_PKGS = autospec clr-bundles common
 

packages

@@ -317,6 +317,7 @@ R-dendextend
 R-desc
 R-devtools
 R-dfoptim
+R-diagram
 R-dichromat
 R-diffobj
 R-diffusionMap
@@ -1035,6 +1036,7 @@ Vulkan-Headers
 Vulkan-Loader
 Vulkan-Tools
 WALinuxAgent
+WPEBackend-fdo
 WireGuard
 XStatic-term.js
 Z3
@@ -1598,6 +1600,7 @@ gnome-music
 gnome-online-accounts
 gnome-panel
 gnome-photos
+gnome-remote-desktop
 gnome-screensaver
 gnome-screenshot
 gnome-session
@@ -2293,6 +2296,7 @@ libwebp
 libwnck
 libwnck3
 libwpd
+libwpe
 libwpg
 libwps
 libxcvt
@@ -2550,6 +2554,7 @@ numactl
 numatop
 numlockx
 numpy-stl
+nv-codec-headers
 nvme-cli
 oath-toolkit
 obs-studio
@@ -3625,6 +3630,7 @@ pyotp
 pyparted
 pypi-absl_py
 pypi-abydos
+pypi-accelerate
 pypi-acme
 pypi-adal
 pypi-aiodns
@@ -3633,6 +3639,7 @@ pypi-aiohttp
 pypi-aiohttp_cors
 pypi-aiosignal
 pypi-aiosmtpd
+pypi-aiosqlite
 pypi-alabaster
 pypi-altgraph
 pypi-amqp
@@ -3667,6 +3674,7 @@ pypi-asyncssh
 pypi-asynctest
 pypi-atomicwrites
 pypi-atpublic
+pypi-attrdict
 pypi-attrs
 pypi-autocommand
 pypi-automat
@@ -3690,6 +3698,7 @@ pypi-bashlex
 pypi-bcrypt
 pypi-beautifulsoup4
 pypi-beniget
+pypi-billiard
 pypi-binaryornot
 pypi-bindep
 pypi-bitmath
@@ -3752,6 +3761,7 @@ pypi-cmarkgfm
 pypi-cmd2
 pypi-codecov
 pypi-colorama
+pypi-coloredlogs
 pypi-colorlog
 pypi-columnize
 pypi-comm
@@ -3790,11 +3800,17 @@ pypi-datetime
 pypi-debugpy
 pypi-decorator
 pypi-deepmerge
+pypi-deepspeed
 pypi-deprecated
 pypi-deprecation
+pypi-devpi_client
+pypi-devpi_common
+pypi-devpi_process
+pypi-devpi_server
 pypi-diff_cover
 pypi-diff_match_patch
 pypi-diffoscope
+pypi-diffusers
 pypi-dill
 pypi-diskcache
 pypi-distlib
@@ -3836,6 +3852,7 @@ pypi-executing
 pypi-expandvars
 pypi-expecttest
 pypi-fabric
+pypi-fairscale
 pypi-falcon
 pypi-fasteners
 pypi-fastimport
@@ -3850,6 +3867,7 @@ pypi-flake8_docstrings
 pypi-flake8_import_order
 pypi-flaky
 pypi-flask
+pypi-flatbuffers
 pypi-flatten_json
 pypi-flit
 pypi-flit_core
@@ -3869,6 +3887,7 @@ pypi-geventhttpclient
 pypi-ghp_import
 pypi-gitdb
 pypi-gitpython
+pypi-glad
 pypi-glfw
 pypi-gmqtt
 pypi-google_api_core
@@ -3909,6 +3928,7 @@ pypi-hatch_requirements_txt
 pypi-hatch_vcs
 pypi-hatchling
 pypi-heapdict
+pypi-hjson
 pypi-hkp4py
 pypi-home_assistant_frontend
 pypi-hpack
@@ -3918,8 +3938,10 @@ pypi-httpcore
 pypi-httplib2
 pypi-httpretty
 pypi-httpx
+pypi-huggingface_hub
 pypi-humanfriendly
 pypi-humanize
+pypi-hupper
 pypi-hyperframe
 pypi-hyperlink
 pypi-hypothesis
@@ -3941,6 +3963,7 @@ pypi-inflect
 pypi-inflection
 pypi-influxdb
 pypi-iniconfig
+pypi-installer
 pypi-intervaltree
 pypi-invoke
 pypi-iocapture
@@ -3991,11 +4014,15 @@ pypi-jupyter_events
 pypi-jupyter_nbgallery
 pypi-jupyter_packaging
 pypi-jupyter_server
+pypi-jupyter_server_fileid
 pypi-jupyter_server_mathjax
 pypi-jupyter_server_terminals
+pypi-jupyter_server_ydoc
 pypi-jupyter_sphinx
+pypi-jupyter_ydoc
 pypi-jupyterlab
 pypi-jupyterlab_launcher
+pypi-jupyterlab_link_share
 pypi-jupyterlab_pygments
 pypi-jupyterlab_server
 pypi-jupyterlab_widgets
@@ -4011,10 +4038,14 @@ pypi-kubernetes
 pypi-langcodes
 pypi-lark_parser
 pypi-latexcodec
+pypi-lavacli
+pypi-lazy
+pypi-lazy_loader
 pypi-lazy_object_proxy
 pypi-ldap3
 pypi-libarchive_c
 pypi-libevdev
+pypi-lightning_utilities
 pypi-linkify_it_py
 pypi-livereload
 pypi-llfuse
@@ -4031,6 +4062,7 @@ pypi-markdown2
 pypi-markdown_it_py
 pypi-matplotlib
 pypi-matplotlib_inline
+pypi-maturin
 pypi-mdit_py_plugins
 pypi-mdurl
 pypi-mergedeep
@@ -4044,6 +4076,7 @@ pypi-mkdocs_exclude
 pypi-mkdocs_material_extensions
 pypi-mkdocstrings
 pypi-mkdocstrings_python
+pypi-ml_dtypes
 pypi-mock
 pypi-modulegraph
 pypi-molecule
@@ -4097,12 +4130,14 @@ pypi-onnx
 pypi-opcodes
 pypi-openshift
 pypi-opt_einsum
+pypi-ordered_set
 pypi-ouimeaux
 pypi-outcome
 pypi-ovirt_imageio
 pypi-packaging
 pypi-paho_mqtt
 pypi-pandocfilters
+pypi-parameterized
 pypi-paramiko
 pypi-parse
 pypi-parse_type
@@ -4110,6 +4145,7 @@ pypi-parsedatetime
 pypi-parsley
 pypi-parso
 pypi-partd
+pypi-passlib
 pypi-paste
 pypi-pastedeploy
 pypi-pastel
@@ -4132,6 +4168,7 @@ pypi-phabricator
 pypi-pickleshare
 pypi-picobox
 pypi-pid
+pypi-pidigits
 pypi-pillow
 pypi-pint
 pypi-pip
@@ -4139,6 +4176,8 @@ pypi-pip_tools
 pypi-pkgconfig
 pypi-pkginfo
 pypi-pkgutil_resolve_name
+pypi-plaster
+pypi-plaster_pastedeploy
 pypi-platformdirs
 pypi-plotly
 pypi-pluggy
@@ -4169,6 +4208,7 @@ pypi-pure_eval
 pypi-purestorage
 pypi-purity_fb
 pypi-py
+pypi-py3nvml
 pypi-py_cpuinfo
 pypi-pyaml
 pypi-pyasn1
@@ -4224,6 +4264,7 @@ pypi-pyqrcode
 pypi-pyqt6_sip
 pypi-pyqt_builder
 pypi-pyquery
+pypi-pyramid
 pypi-pyreadline
 pypi-pyrfc3339
 pypi-pyro4
@@ -4277,6 +4318,7 @@ pypi-python_version
 pypi-python_xlib
 pypi-pythran
 pypi-pytoolconfig
+pypi-pytorch_lightning
 pypi-pytz
 pypi-pytz_deprecation_shim
 pypi-pyu2f
@@ -4313,6 +4355,7 @@ pypi-requirements_parser
 pypi-resolvelib
 pypi-responses
 pypi-restructuredtext_lint
+pypi-retrolab
 pypi-retry
 pypi-retry_decorator
 pypi-retryz
@@ -4348,6 +4391,7 @@ pypi-selenium
 pypi-selinux
 pypi-semantic_version
 pypi-send2trash
+pypi-sentencepiece
 pypi-sentinels
 pypi-sentry_sdk
 pypi-seqdiag
@@ -4416,7 +4460,9 @@ pypi-sshtunnel
 pypi-stack_data
 pypi-statsmodels
 pypi-storops
+pypi-strictyaml
 pypi-subprocess_tee
+pypi-suds
 pypi-sure
 pypi-svg.path
 pypi-tabulate
@@ -4438,17 +4484,20 @@ pypi-three_merge
 pypi-tifffile
 pypi-tinycss2
 pypi-tldextract
+pypi-tokenizers
 pypi-toml
 pypi-tomli
 pypi-tomli_w
 pypi-tomlkit
 pypi-toolz
+pypi-torchmetrics
 pypi-tornado
 pypi-tornado_xstatic
 pypi-tox
 pypi-tqdm
 pypi-traitlets
 pypi-trampolim
+pypi-translationstring
 pypi-trimesh
 pypi-trio
 pypi-trio_websocket
@@ -4496,6 +4545,9 @@ pypi-urwid
 pypi-user_agents
 pypi-userpath
 pypi-uvloop
+pypi-venusian
+pypi-verboselogs
+pypi-versioneer
 pypi-vine
 pypi-virtualenv
 pypi-volatile
@@ -4521,6 +4573,7 @@ pypi-wrapt
 pypi-wsgiproxy2
 pypi-wsproto
 pypi-wurlitzer
+pypi-xgboost
 pypi-xmldiff
 pypi-xmljson
 pypi-xmlschema
@@ -4540,11 +4593,14 @@ pypi-xstatic_objectpath
 pypi-xstatic_roboto_fontface
 pypi-xvfbwrapper
 pypi-xyzservices
+pypi-y_py
 pypi-yamllint
 pypi-yamlloader
 pypi-yapf
 pypi-yarg
 pypi-yarl
+pypi-yattag
+pypi-ypy_websocket
 pypi-zabbix_api
 pypi-zc.lockfile
 pypi-zeroconf
@@ -4668,6 +4724,7 @@ rest
 restic
 rhythmbox
 rinutils
+ripgrep
 rlwrap
 rng-tools
 robin-map
@@ -4712,6 +4769,7 @@ seahorse
 seatd
 sed
 seer
+sentencepiece
 serf
 setserial
 setxkbmap
@@ -4797,7 +4855,6 @@ sympy
 syndication
 synergy
 syntax-highlighting
-sysdig
 syslinux
 sysprof
 sysstat
@@ -4856,6 +4913,7 @@ tracker
 tracker-miners
 tree
 tree-sitter
+trurl
 tzdata
 uchardet
 ucpp
@@ -4913,6 +4971,7 @@ wayland-protocols
 wcslib
 webkitgtk
 webkitgtk50
+webkitgtk60
 weechat
 weston
 wget

vendor.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+from git import Repo
+
+import requests
+
+
+def vendor_check():
+    if not os.path.isfile('options.conf'):
+        return False
+    config = configparser.ConfigParser(interpolation=None)
+    config.read('options.conf')
+    if 'autospec' not in config.sections():
+        return False
+    if vendor := config['autospec'].get('cargo_vendor'):
+        if vendor == "true":
+            return 'cargo'
+
+    return False
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('url')
+    parser.add_argument('name')
+    parser.add_argument('git')
+    return parser.parse_args()
+
+
+def setup_content(url):
+    tdir = tempfile.mkdtemp()
+    outfile = os.path.join(tdir, os.path.basename(url))
+    response = requests.get(url, timeout=30)
+    response.raise_for_status()
+    with open(outfile, 'wb') as cfile:
+        cfile.write(response.content)
+
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    os.remove(outfile)
+    return tdir
+
+
+def setup_cargo_vendor(path):
+    for dirpath, _, files in os.walk(path):
+        for fname in files:
+            if fname == "Cargo.toml":
+                return dirpath
+    return False
+
+
+def update_cargo_vendor(path, name, git):
+    git_uri = os.path.join(git, name)
+    vendor_path = os.path.join(path, 'vendor')
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    vendor_git = os.path.join(vendor_path, '.git')
+    if not os.path.isdir(vendor_git):
+        # initialize a git repo
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
+                       shell=True, check=True)
+    backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
+    subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
+                   shell=True, check=True)
+    shutil.rmtree(vendor_path)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
+    subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
+                   shell=True, check=True)
+    repo = Repo(vendor_path)
+    if not len(repo.untracked_files) > 0 or repo.is_dirty():
+        return False
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
+                   shell=True, check=True)
+    gmt = time.gmtime()
+    tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
+    subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
+                   check=True)
+    subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
+                   shell=True, check=True)
+    time.sleep(30)
+    return tag
+
+
+def update_cargo_sources(name, tag):
+    makefile = []
+    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                 'snapshot', name)
+    with open('Makefile', encoding='utf8') as mfile:
+        for line in mfile.readlines():
+            if line.startswith('ARCHIVES'):
+                if re.match(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                else:
+                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                makefile.append(new_archives)
+            else:
+                makefile.append(line)
+    with open('Makefile', 'w', encoding='utf8') as mfile:
+        mfile.writelines(makefile)
+
+
+def main():
+    vtype = vendor_check()
+    if not vtype:
+        return
+
+    args = get_args()
+    tdir = setup_content(args.url)
+    if vtype == 'cargo':
+        vdir = setup_cargo_vendor(tdir)
+        if vdir:
+            tag = update_cargo_vendor(vdir, args.name, args.git)
+            if tag:
+                update_cargo_sources(args.name, tag)
+    shutil.rmtree(tdir)
+
+if __name__ == '__main__':
+    main()