packages update

Changes vendor from a target (it could still have a stand-alone target
but it wouldn't be used by the autospec target).

This change is needed because the Makefile is rewritten in the
autospec target but the ARCHIVES make variable is going to use the
value when make is executed. The common change causes the ARCHIVES
value to be eval'd during the autospec target execution in order for
the update from the vendor script to be seen by make.

The vendor script changes fix bugs in updating the options.conf and
Makefile. It also now handles returning the original ARCHIVES value as
well as prevent subshell scripts from outputting and adding badness
into the eval'd ARCHIVES variable.

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -157,10 +157,6 @@ pullrebase:
 		fi \
 	fi
 
-# help vendor: Attempt to create an updated vendor tar archive if needed
-vendor: preautospec-checks pullrebase clean-old-content
-	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
-
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -174,8 +170,8 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
-	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

packages

@@ -43,6 +43,7 @@ OpenSP
 PDAL
 PyMySQL
 PyQt5
+PyQt6
 PySocks
 PyYAML
 QAT-ZSTD-Plugin
@@ -109,6 +110,7 @@ R-ISOcodes
 R-ISwR
 R-Iso
 R-JM
+R-KMsurv
 R-Lahman
 R-LearnBayes
 R-Luminescence
@@ -375,6 +377,7 @@ R-estimability
 R-etm
 R-evaluate
 R-evd
+R-exactRankTests
 R-expint
 R-expm
 R-expsmooth
@@ -413,6 +416,7 @@ R-fixest
 R-flashClust
 R-flexmix
 R-flexsurv
+R-flexsurvcure
 R-flextable
 R-float
 R-fontBitstreamVera
@@ -460,9 +464,13 @@ R-ggfortify
 R-ggjoy
 R-ggplot2
 R-ggplot2movies
+R-ggpubr
 R-ggrepel
 R-ggridges
+R-ggsci
+R-ggsignif
 R-ggstats
+R-ggtext
 R-ggvis
 R-gh
 R-git2r
@@ -482,6 +490,7 @@ R-gower
 R-gplots
 R-gridBase
 R-gridExtra
+R-gridtext
 R-gsl
 R-gss
 R-gtable
@@ -536,6 +545,7 @@ R-kernlab
 R-kimisc
 R-kit
 R-klaR
+R-km.ci
 R-kmi
 R-knitr
 R-ks
@@ -596,6 +606,7 @@ R-mathjaxr
 R-matrixStats
 R-matrixcalc
 R-maxLik
+R-maxstat
 R-mclogit
 R-mclust
 R-mcmc
@@ -842,6 +853,7 @@ R-rsdmx
 R-rstan
 R-rstanarm
 R-rstantools
+R-rstatix
 R-rstpm2
 R-rstudioapi
 R-rsvg
@@ -922,7 +934,9 @@ R-styler
 R-subplex
 R-subselect
 R-superpc
+R-survMisc
 R-survey
+R-survminer
 R-svUnit
 R-svglite
 R-sys
@@ -1100,7 +1114,6 @@ ark
 armadillo
 arp-scan
 arpack-ng
-artikulate
 asciidoc
 asciidoctor
 asciinema
@@ -1175,6 +1188,7 @@ boto3
 botocore
 bovo
 box2d
+bpftool
 bpftrace
 breeze
 breeze-gtk
@@ -1218,7 +1232,6 @@ calendarsupport
 can-utils
 cantarell-fonts
 cantata
-cantor
 capnproto
 capslock
 capstone
@@ -1434,6 +1447,7 @@ dunst
 duperemove
 dvisvgm
 dwarves
+dymo-cups-drivers
 dyskctl
 e2fsprogs
 earlyoom
@@ -1484,6 +1498,7 @@ fann
 farstream
 faultstat
 fcgi
+fdk-aac
 fdupes
 feh
 fetchmail
@@ -1501,6 +1516,7 @@ fish
 flac
 flare-engine
 flare-game
+flatpack-kcm
 flatpak
 flatpak-builder
 fldigi
@@ -1542,7 +1558,6 @@ gc
 gcab
 gcc
 gcc11
-gcc14
 gcc7
 gcc8
 gcc9
@@ -1707,6 +1722,7 @@ gtk3
 gtk4
 gtkmm2
 gtkmm3
+gtkmm4
 gtksourceview
 gtksourceview4
 gtkspell3
@@ -1715,7 +1731,6 @@ guile
 gutenprint
 gvfs
 gvim
-gwenview
 gzip
 hamlib
 haproxy
@@ -1827,8 +1842,6 @@ json-glib
 jsoncpp
 juk
 kaccounts-integration
-kactivities
-kactivities-stats
 kactivitymanagerd
 kalarm
 kalgebra
@@ -1845,7 +1858,6 @@ kbackup
 kbd
 kblackbox
 kblocks
-kblog
 kbookmarks
 kbounce
 kbreakout
@@ -1859,6 +1871,7 @@ kcmutils
 kcodecs
 kcolorchooser
 kcolorpicker
+kcolorscheme
 kcompletion
 kconfig
 kconfigwidgets
@@ -1880,27 +1893,24 @@ kded
 kdeedu-data
 kdegraphics-mobipocket
 kdegraphics-thumbnailers
-kdelibs4support
 kdenetwork-filesharing
 kdenlive
 kdepim-runtime
 kdeplasma-addons
 kdesdk-kio
 kdesdk-thumbnailers
-kdesignerplugin
 kdesu
 kdf
 kdiagram
 kdialog
 kdiamond
-kdiff3
 kdnssd
 kdoctools
 kdsoap
+kdsoap-ws-discovery-client
 keditbookmarks
 keepalived
 keepassxc
-kemoticons
 kernel-config
 kernel-install
 kexec-tools
@@ -1908,27 +1918,24 @@ keychain
 keyutils
 kfilemetadata
 kfind
-kfloppy
 kfourinline
-kgamma5
+kgamma
 kgeography
 kget
 kglobalaccel
+kglobalacceld
 kgoldrunner
 kgpg
 kguiaddons
 khangman
 khelpcenter
 kholidays
-khotkeys
-khtml
 ki18n
 kicad
 kicad-packages3D
 kiconthemes
 kidentitymanagement
 kidletime
-kig
 kigo
 killbots
 kimageannotator
@@ -1936,11 +1943,11 @@ kimageformats
 kimagemapeditor
 kimap
 kinfocenter
-kinit
 kio
 kio-admin
 kio-extras
 kio-zeroconf
+kirigami
 kirigami-addons
 kirigami-gallery
 kirigami2
@@ -1951,7 +1958,6 @@ kiten
 kitinerary
 kjobwidgets
 kjs
-kjsembed
 kjumpingcube
 kldap
 kleopatra
@@ -1964,11 +1970,9 @@ kmail
 kmail-account-wizard
 kmailtransport
 kmbox
-kmediaplayer
 kmenuedit
 kmime
 kmines
-kmix
 kmod
 kmousetool
 kmouth
@@ -1985,6 +1989,7 @@ koji
 kolf
 kollision
 kolourpaint
+kompare
 konqueror
 konquest
 konsole
@@ -2005,10 +2010,8 @@ krb5
 krdc
 kreversi
 kronosnet
-kross
 kruler
 krunner
-krusader
 ksanecore
 kscreen
 kscreenlocker
@@ -2021,9 +2024,9 @@ ksnakeduel
 kspaceduel
 ksquares
 ksshaskpass
-kstars
+kstatusnotifieritem
 ksudoku
-ksysguard
+ksvg
 ksystemlog
 ksystemstats
 kteatime
@@ -2033,7 +2036,6 @@ ktexttemplate
 ktextwidgets
 ktimer
 ktnef
-ktp-desktop-applets
 ktuberling
 kturtle
 kubernetes
@@ -2044,7 +2046,6 @@ kwallet
 kwallet-pam
 kwalletmanager
 kwayland
-kwayland-integration
 kwayland-server
 kwidgetsaddons
 kwin
@@ -2052,11 +2053,9 @@ kwindowsystem
 kwordquiz
 kwrited
 kxmlgui
-kxmlrpcclient
 kyotocabinet
 ladspa_sdk
 latencytop
-latte-dock
 layer-shell-qt
 lcms2
 lcov
@@ -2103,6 +2102,7 @@ libatomic_ops
 libavif
 libavtp
 libb64
+libbacktrace
 libblockdev
 libbpf
 libbsd
@@ -2207,7 +2207,6 @@ libkdepim
 libkeduvocdocument
 libkexiv2
 libkgapi
-libkipi
 libkleo
 libkmahjongg
 libkomparediff2
@@ -2230,6 +2229,7 @@ libmodplug
 libmodulemd
 libmpack
 libmpack-lua
+libmsgraph
 libmspack
 libmspub
 libmtp
@@ -2266,6 +2266,7 @@ libpeas
 libpinyin
 libpipeline
 libplacebo
+libplasma
 libplist
 libpng
 libportal
@@ -2402,7 +2403,6 @@ lm-sensors
 lmdb
 logrotate
 logutils
-lokalize
 lpeg
 lsacpi
 lshw
@@ -2458,7 +2458,6 @@ meson
 messagelib
 meta-c-basic
 meta-desktop-gnome
-meta-desktop-kde
 meta-os-core
 meta-os-core-plus
 metacity
@@ -2471,6 +2470,7 @@ micro-config-drive-aws
 micro-config-drive-equinix
 micro-config-drive-oci
 milou
+mimetreeparser
 minetest
 minetest_game
 mingw-binutils
@@ -2606,7 +2606,6 @@ ocamlbuild
 ocl-icd
 octave
 oiio
-okteta
 okular
 oneDPL
 oneVPL
@@ -2650,7 +2649,7 @@ osm-gps-map
 ostree
 otp
 oxygen
-oxygen-icons5
+oxygen-icons
 oxygen-sounds
 p11-kit
 p7zip
@@ -3604,18 +3603,23 @@ pipewire
 pixman
 pkcs11-helper
 pkg-config
+plasma-activities
+plasma-activities-stats
 plasma-browser-integration
 plasma-desktop
 plasma-discover
-plasma-framework
+plasma-disks
+plasma-firewall
 plasma-integration
 plasma-nm
 plasma-pa
 plasma-systemmonitor
 plasma-vault
 plasma-wayland-protocols
+plasma-welcome
 plasma-workspace
 plasma-workspace-wallpapers
+plasma5support
 plzip
 pm-graph
 pmdk
@@ -3627,6 +3631,7 @@ podman
 polkit
 polkit-kde-agent
 polkit-qt
+polkit-qt6
 poppler
 poppler-data-clr-rename
 popt
@@ -3660,7 +3665,9 @@ psstop
 pth
 pugixml
 pulseaudio
+pulseaudio-qt
 purpose
+puzzles
 pv
 pwgen
 pyatspi
@@ -3787,6 +3794,7 @@ pypi-certifi
 pypi-cffi
 pypi-cffsubr
 pypi-cfgv
+pypi-chai
 pypi-chaospy
 pypi-chardet
 pypi-charset_normalizer
@@ -3881,6 +3889,8 @@ pypi-docopt
 pypi-docstring_parser
 pypi-docstring_to_markdown
 pypi-docutils
+pypi-doxypypy
+pypi-doxyqml
 pypi-dpcontracts
 pypi-dulwich
 pypi-ecdsa
@@ -3981,6 +3991,7 @@ pypi-hatch_requirements_txt
 pypi-hatch_vcs
 pypi-hatchling
 pypi-heapdict
+pypi-hishel
 pypi-hjson
 pypi-hkp4py
 pypi-home_assistant_frontend
@@ -4088,6 +4099,7 @@ pypi-kiwisolver
 pypi-krb5
 pypi-kubernetes
 pypi-langcodes
+pypi-language_data
 pypi-lark_parser
 pypi-latexcodec
 pypi-lavacli
@@ -4113,6 +4125,7 @@ pypi-lxml
 pypi-lz4
 pypi-macholib
 pypi-mako
+pypi-marisa_trie
 pypi-markdown
 pypi-markdown2
 pypi-markdown_exec
@@ -4132,6 +4145,7 @@ pypi-mkdocs
 pypi-mkdocs_autorefs
 pypi-mkdocs_exclude
 pypi-mkdocs_gen_files
+pypi-mkdocs_get_deps
 pypi-mkdocs_htmlproofer_plugin
 pypi-mkdocs_material_extensions
 pypi-mkdocs_monorepo_plugin
@@ -4179,6 +4193,7 @@ pypi-nocasedict
 pypi-nocaselist
 pypi-nodeenv
 pypi-notebook_shim
+pypi-numexpr
 pypi-numpoly
 pypi-numpy
 pypi-numpydoc
@@ -4196,6 +4211,7 @@ pypi-outcome
 pypi-overrides
 pypi-ovirt_imageio
 pypi-packaging
+pypi-packaging_legacy
 pypi-paginate
 pypi-paho_mqtt
 pypi-pandocfilters
@@ -4216,6 +4232,7 @@ pypi-pathlib2
 pypi-pathspec
 pypi-patsy
 pypi-pbr
+pypi-pbs_installer
 pypi-pdfminer.six
 pypi-pdm
 pypi-pdm_backend
@@ -4260,6 +4277,7 @@ pypi-portend
 pypi-posix_ipc
 pypi-poyo
 pypi-pre_commit
+pypi-pretty_errors
 pypi-prettytable
 pypi-probed
 pypi-progress
@@ -4307,6 +4325,7 @@ pypi-pyelftools
 pypi-pyfakefs
 pypi-pyflakes
 pypi-pygame
+pypi-pygdbmi
 pypi-pygithub
 pypi-pygments
 pypi-pyicu
@@ -4454,6 +4473,7 @@ pypi-safetensors
 pypi-salib
 pypi-sanic_routing
 pypi-scikit_build
+pypi-scikit_build_core
 pypi-scikit_image
 pypi-scikit_learn
 pypi-scp
@@ -4533,6 +4553,7 @@ pypi-sqlalchemy
 pypi-sqlparse
 pypi-sshtunnel
 pypi-stack_data
+pypi-starlette
 pypi-storops
 pypi-strictyaml
 pypi-subprocess_tee
@@ -4582,6 +4603,7 @@ pypi-ttp
 pypi-twine
 pypi-typed_ast
 pypi-typeguard
+pypi-types_cffi
 pypi-types_colorama
 pypi-types_cryptography
 pypi-types_dataclasses
@@ -4625,6 +4647,7 @@ pypi-urlobject
 pypi-urwid
 pypi-user_agents
 pypi-userpath
+pypi-uvicorn
 pypi-uvloop
 pypi-venusian
 pypi-verboselogs
@@ -4638,6 +4661,7 @@ pypi-vsts_cd_manager
 pypi-waitress
 pypi-warlock
 pypi-watchdog
+pypi-watchfiles
 pypi-wbdata
 pypi-wcmatch
 pypi-wcwidth
@@ -4734,6 +4758,7 @@ qbittorrent
 qca
 qca-qt5
 qcoro
+qcoro6
 qemu
 qemu-guest-additions
 qgit
@@ -4851,7 +4876,6 @@ rlwrap
 rng-tools
 robin-map
 roboto
-rocs
 rofi
 rpcbind
 rpcsvc-proto
@@ -4866,6 +4890,7 @@ rtmpdump
 rttr
 ruby
 runc
+rust-bindgen
 rustc
 rxvt-unicode
 s2tc
@@ -5062,7 +5087,6 @@ usb-modeswitch
 usb-modeswitch-data
 usbredir
 usbutils
-user-manager
 userspace-rcu
 usrbinjava
 usrbinvi

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:

vendor.py

@@ -30,6 +30,7 @@ def vendor_check():
 
 def get_args():
     parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
     parser.add_argument('url')
     parser.add_argument('name')
     parser.add_argument('git')
@@ -44,7 +45,7 @@ def setup_content(url):
     with open(outfile, 'wb') as cfile:
         cfile.write(response.content)
 
-    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
     os.remove(outfile)
     return tdir
 
@@ -60,61 +61,88 @@ def setup_cargo_vendor(path):
 def update_cargo_vendor(path, name, git):
     git_uri = os.path.join(git, name)
     vendor_path = os.path.join(path, 'vendor')
-    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
     if not os.path.isdir(vendor_git):
         # initialize a git repo
-        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
-                       shell=True, check=True)
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
     backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
     subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
         return False
-    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     gmt = time.gmtime()
     tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
     subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
-                   check=True)
+                   check=True, stdout=subprocess.DEVNULL)
     subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     time.sleep(30)
     return tag
 
 
 def update_cargo_sources(name, tag):
     makefile = []
-    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+    options = []
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
                                  'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.match(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
                     new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                                          f"{archive_replace}-{tag}.tar.xz", line)
                 else:
-                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
                 makefile.append(line)
     with open('Makefile', 'w', encoding='utf8') as mfile:
         mfile.writelines(makefile)
 
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+
 
 def main():
+    updated = False
+    args = get_args()
+
     vtype = vendor_check()
     if not vtype:
+        print(args.archives)
         return
 
-    args = get_args()
     tdir = setup_content(args.url)
     if vtype == 'cargo':
         vdir = setup_cargo_vendor(tdir)
@@ -122,7 +150,11 @@ def main():
             tag = update_cargo_vendor(vdir, args.name, args.git)
             if tag:
                 update_cargo_sources(args.name, tag)
+                updated = True
+    if not updated:
+        print(args.archives)
     shutil.rmtree(tdir)
 
+
 if __name__ == '__main__':
     main()