From dcc38f219381e0ead09023afa5fad148858ff4d5 Mon Sep 17 00:00:00 2001 From: William Douglas Date: Thu, 23 Mar 2023 14:55:41 -0700 Subject: [PATCH] Add vendor target Add handling for a vendor target that will handle specific vendor requirements from a package's options.conf. The initial change adds handling for 'cargo vendor', creating a new repo in the configured git repositiory and setting up the ARCHIVES variable in the package's Makefile. If the repo already exists, the tooling will check if a repo change was detected after re-vendoring the sources. If a change is detected the new content will be pushed and the ARCHIVES variable will be updated. Signed-off-by: William Douglas --- Makefile.common | 5 ++ Makefile.config | 3 ++ vendor.py | 123 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100755 vendor.py diff --git a/Makefile.common b/Makefile.common index f8fe49f..8bbef21 100644 --- a/Makefile.common +++ b/Makefile.common @@ -157,6 +157,10 @@ pullrebase: fi \ fi +# help vendor: Attempt to create an updated vendor tar archive if needed +vendor: preautospec-checks pullrebase clean-old-content + @ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL) + preautospec-checks: #help autospec: automatically generates a specfile. If there is @@ -171,6 +175,7 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content exit 1; \ fi @printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile + @$(MAKE) vendor python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \ --target . \ --integrity \ diff --git a/Makefile.config b/Makefile.config index 5711e9e..86b586c 100644 --- a/Makefile.config +++ b/Makefile.config @@ -35,6 +35,9 @@ PKG_BASE_URL = https://github.com/clearlinux-pkgs # Location where project repos are hosted. PRJ_BASE_URL = https://github.com/clearlinux +# Location where vendor repos are hosted. +VND_BASE_URL = + # Project repos that are integral to the common tooling. PROJ_PKGS = autospec clr-bundles common diff --git a/vendor.py b/vendor.py new file mode 100755 index 0000000..c183104 --- /dev/null +++ b/vendor.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import argparse +import configparser +import os +import re +import shutil +import subprocess +import tempfile +import time + +from git import Repo + +import requests + + +def vendor_check(): + config = configparser.ConfigParser(interpolation=None) + config.read('options.conf') + if config['autospec'].get('cargo_vendor'): + return 'cargo' + else: + return False + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('url') + parser.add_argument('name') + parser.add_argument('git') + return parser.parse_args() + + +def setup_content(url): + tdir = tempfile.mkdtemp() + outfile = os.path.join(tdir, os.path.basename(url)) + response = requests.get(url, timeout=30) + response.raise_for_status() + with open(outfile, 'wb') as cfile: + cfile.write(response.content) + + subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True) + os.remove(outfile) + return tdir + + +def setup_cargo_vendor(path): + for dirpath, _, files in os.walk(path): + for fname in files: + if fname == "Cargo.toml": + return dirpath + return False + + +def update_cargo_vendor(path, name, git): + git_uri = os.path.join(git, name) + vendor_path = os.path.join(path, 'vendor') + subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True) + vendor_git = os.path.join(vendor_path, '.git') + if not os.path.isdir(vendor_git): + # initialize a git repo + subprocess.run('git init .', cwd=vendor_path, shell=True, check=True) + subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path, + shell=True, check=True) + backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git') + subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path, + shell=True, check=True) + shutil.rmtree(vendor_path) + subprocess.run('cargo vendor', cwd=path, shell=True, check=True) + subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path, + shell=True, check=True) + repo = Repo(vendor_path) + if not len(repo.untracked_files) > 0 or repo.is_dirty(): + return False + subprocess.run('git add .', cwd=vendor_path, shell=True, check=True) + subprocess.run('git commit -m "vendor update"', cwd=vendor_path, + shell=True, check=True) + gmt = time.gmtime() + tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}" + subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True, + check=True) + subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path, + shell=True, check=True) + time.sleep(30) + return tag + + +def update_cargo_sources(name, tag): + makefile = [] + archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name, + 'snapshot', name) + with open('Makefile', encoding='utf8') as mfile: + for line in mfile.readlines(): + if line.startswith('ARCHIVES'): + if re.match(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', line): + new_archives = re.sub(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', + f"{archive_match}-{tag}.tar.xz\n", line) + else: + new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n" + makefile.append(new_archives) + else: + makefile.append(line) + with open('Makefile', 'w', encoding='utf8') as mfile: + mfile.writelines(makefile) + + +def main(): + vtype = vendor_check() + if not vtype: + return + + args = get_args() + tdir = setup_content(args.url) + if vtype == 'cargo': + vdir = setup_cargo_vendor(tdir) + if vdir: + tag = update_cargo_vendor(vdir, args.name, args.git) + if tag: + update_cargo_sources(args.name, tag) + shutil.rmtree(tdir) + +if __name__ == '__main__': + main()