Add github url handling for tags with slashes

vectorscan uses 'vectorscan/' as a tag prefix which confuses our
version parsing (even more so with v being the tag prefix). Add
support for matching any '*/version' tag names as the top priority.

Signed-off-by: William Douglas <william.douglas@intel.com>

README.rst

@@ -268,6 +268,12 @@ prep_prepend
   resulting ``.spec``, and is used for situations where fine-grained
   control is required.
 
+copy_prepend
+  Additional actions that should take place directly before the source
+  directory is copied for other builds (32bit, avx2, etc). This will be
+  placed in the resulting ``.spec``, and is used for situations where
+  fine-grained control is required.
+
 build_prepend
   Additional actions that should take place after ``%build`` and before
   the ``%configure`` macro or equivalent (``%cmake``, etc.). If autospec

autospec/abireport.py

@@ -163,7 +163,7 @@ def dump_symbols(path):
     try:
         lines = get_output(cmd)
     except Exception as e:
-        print("Fatal error inspecting {}: {}".format(path, e))
+        util.print_fatal("Fatal error inspecting {}: {}".format(path, e))
         sys.exit(1)
     for line in lines.split("\n"):
         line = line.strip()
@@ -236,6 +236,7 @@ def examine_abi_host(download_path, results_dir, name):
                   cwd=download_path)
     except Exception as e:
         util.print_fatal("Error invoking abireport: {}".format(e))
+        sys.exit(1)
 
 
 def examine_abi_fallback(download_path, results_dir, name):
@@ -272,6 +273,7 @@ def examine_abi_fallback(download_path, results_dir, name):
             subprocess.check_call(cmd, shell=True)
     except Exception as e:
         util.print_fatal("Error extracting RPMS: {}".format(e))
+        sys.exit(1)
 
     os.chdir(download_path)
     collected_files = set()

autospec/autospec.py

@@ -38,7 +38,7 @@ import specfiles
 import tarball
 from abireport import examine_abi
 from logcheck import logcheck
-from util import binary_in_path, print_fatal, write_out
+from util import binary_in_path, print_build_failed, print_fatal, write_out
 
 sys.path.append(os.path.dirname(__file__))
 
@@ -269,9 +269,13 @@ def package(args, url, name, archives, workingdir):
         pkg_integrity.check(url, conf, interactive=interactive_mode)
         pkg_integrity.load_specfile(specfile)
 
-    specfile.write_spec()
+    spec_type = specfile.write_spec()
+
     while 1:
         package.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, content, args.cleanup)
+        if spec_type == "template":
+            # specfile template is assumed "correct" and any failures need to be manually addressed
+            break
         filemanager.load_specfile(specfile)
         specfile.write_spec()
         filemanager.newfiles_printed = 0
@@ -291,7 +295,7 @@ def package(args, url, name, archives, workingdir):
 
     if package.success == 0:
         conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
-        print_fatal("Build failed, aborting")
+        print_build_failed()
         sys.exit(1)
     elif os.path.isfile("README.clear"):
         try:
@@ -304,7 +308,8 @@ def package(args, url, name, archives, workingdir):
         except Exception:
             pass
 
-    check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
+    if spec_type == "generate":
+        check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
 
     examine_abi(conf.download_path, content.name)
     if os.path.exists("/var/lib/rpm"):

autospec/build.py

@@ -22,6 +22,7 @@
 import os
 import re
 import shutil
+import sys
 
 import util
 
@@ -153,7 +154,7 @@ class Build(object):
             return True
         self.must_restart = 0
         self.file_restart = 0
-        is_clean = True
+        fatals = []
         util.call("sync")
         with util.open_auto(filename, "r") as rootlog:
             loglines = rootlog.readlines()
@@ -161,9 +162,10 @@ class Build(object):
         for line in loglines:
             match = missing_pat.match(line)
             if match is not None:
-                util.print_fatal("Cannot resolve dependency name: {}".format(match.group(1)))
-                is_clean = False
-        return is_clean
+                fatals.append(f"Cannot resolve dependency name: {match.group(1)}")
+        if fatals:
+            util.print_fatal('\n'.join(fatals))
+            sys.exit(1)
 
     def parse_build_results(self, filename, returncode, filemanager, config, requirements, content):
         """Handle build log contents."""
@@ -283,7 +285,7 @@ class Build(object):
             mockopts,
         ]
 
-        if not cleanup and self.must_restart == 0 and self.file_restart > 0 and set(filemanager.excludes) == set(filemanager.manual_excludes):
+        if config.config_opts.get('avoid_rebuild') and not cleanup and self.must_restart == 0 and self.file_restart > 0 and set(filemanager.excludes) == set(filemanager.manual_excludes):
             cmd_args.append("--no-clean")
             cmd_args.append("--short-circuit=binary")
 
@@ -295,12 +297,11 @@ class Build(object):
         # sanity check the build log
         if not os.path.exists(config.download_path + "/results/build.log"):
             util.print_fatal("Mock command failed, results log does not exist. User may not have correct permissions.")
-            exit(1)
+            sys.exit(1)
 
-        if not self.parse_buildroot_log(config.download_path + "/results/root.log", ret):
-            return
+        self.parse_buildroot_log(config.download_path + "/results/root.log", ret)
 
         self.parse_build_results(config.download_path + "/results/build.log", ret, filemanager, config, requirements, content)
         if filemanager.has_banned:
             util.print_fatal("Content in banned paths found, aborting build")
-            exit(1)
+            sys.exit(1)

autospec/buildreq.py

@@ -147,7 +147,7 @@ def parse_modules_list(modules_string, is_cmake=False):
         modules = [m for m in re.split(r'\s*([><]?=|\${?[^}]*}?)\s*', modules_string)]
         modules = filter(None, modules)
     else:
-        modules = [m.strip('[]') for m in modules_string.split()]
+        modules = [m.strip('[]').strip('"') for m in modules_string.split()]
     res = []
     next_is_ver = False
     for mod in modules:

autospec/check.py

@@ -97,17 +97,23 @@ def scan_for_tests(src_dir, config, requirements, content):
     }
     if config.config_opts.get('32bit'):
         testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../build32/clr-build32;\n" + cmake_check + " || :"
         testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
     if config.config_opts.get('use_avx2'):
         testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx2/clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx2;\n" + meson_check + " || :"
     if config.config_opts.get('use_avx512'):
         testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx512/clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx512;\n" + meson_check + " || :"
+    if config.config_opts.get('use_apx'):
+        testsuites["makecheck"] += "\ncd ../buildapx;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildapx/clr-build-apx;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildapx;\n" + meson_check + " || :"
     if config.config_opts.get('openmpi'):
         testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
-        testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi
+        testsuites["cmake"] += "\ncd ../../build-openmpi/clr-build-openmpi;\n" + cmake_check_openmpi
 
     files = os.listdir(src_dir)
 

autospec/commitmessage.py

@@ -49,7 +49,7 @@ def scan_for_changes(download_path, directory, transforms):
                 shutil.copy(source, target)
                 os.chmod(target, 0o644)
             except Exception as e:
-                print("Error copying file: {}".format(e))
+                util.print_fatal("Error copying file: {}".format(e))
                 sys.exit(1)
             found.append(item)
 

autospec/config.py

@@ -28,7 +28,8 @@ from collections import OrderedDict
 
 import check
 import license
-from util import call, open_auto, print_info, print_warning, write_out
+from util import (call, open_auto, print_fatal, print_info, print_warning,
+                  write_out)
 
 
 def read_pattern_conf(filename, dest, list_format=False, path=None):
@@ -68,6 +69,7 @@ class Config(object):
 
     def __init__(self, download_path):
         """Initialize Default configuration settings."""
+        self.cargo_vendors = ""
         self.content = None  # hack to avoid circular init dependency
         self.extra_configure = ""
         self.extra_configure32 = ""
@@ -91,6 +93,7 @@ class Config(object):
         self.install_macro = "%make_install"
         self.disable_static = "--disable-static"
         self.prep_prepend = []
+        self.copy_prepend = []
         self.build_prepend = []
         self.build_prepend_once = []
         self.build_append = []
@@ -154,6 +157,7 @@ class Config(object):
         }
         self.config_opts = {}
         self.config_options = {
+            "avoid_rebuild": "Try to use mock short circuit to avoid full rebuilds",
             "broken_c++": "extend flags with '-std=gnu++98",
             "cargo_vendor": "create vendor archive with cargo",
             "use_lto": "configure build for lto",
@@ -267,6 +271,7 @@ class Config(object):
             (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC \(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'),
             (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None),
             (r"Checking for (.*?)\.\.\.no", 0, None),
+            (r"checking for (.*?) \(using pkg-config\)\.\.\. no", 0, None),
             (r"Checking for (.*?)\s*: not found", 0, None),
             (r"Checking for (.*?)\s>=.*\s*: not found", 0, None),
             (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None),
@@ -289,10 +294,12 @@ class Config(object):
             (r"No rule to make target `(.*)',", 0, None),
             (r"Package (.*) was not found in the pkg-config search path.", 0, 'pkgconfig'),
             (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'),
+            (r"The file `([a-zA-Z0-9\-:]*)\.pc` needs to be installed and the PKG_CONFIG_PATH environment variable must contain its parent directory\.", 0, 'pkgconfig'),
             (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'),
             (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'),
             (r"Program (.*) found: NO", 0, None),
             (r"Target '[a-zA-Z0-9\-]' can't be generated as '(.*)' could not be found", 0, None),
+            (r"The missing Perl modules are:\s*(\S+)", 0, 'perl'),
             (r"Unable to `import (.*)`", 0, None),
             (r"Unable to find '(.*)'", 0, None),
             (r"Warning: prerequisite ([a-zA-Z:]+) [0-9\.]+ not found.", 0, 'perl'),
@@ -435,20 +442,24 @@ class Config(object):
         # next the options
         config_f['autospec'] = {}
         for fname, comment in sorted(self.config_options.items()):
+            fpath = os.path.join(self.download_path, fname)
             config_f.set('autospec', '# {}'.format(comment))
-            if os.path.exists(fname):
+            if os.path.isfile(fpath):
                 config_f['autospec'][fname] = 'true'
-                os.remove(fname)
+                os.remove(fpath)
             else:
                 config_f['autospec'][fname] = 'false'
 
         # default lto to true for new things
         config_f['autospec']['use_lto'] = 'true'
+        # default autoupdate to true for new things
+        config_f['autospec']['autoupdate'] = 'true'
 
         # renamed options need special care
-        if os.path.exists("skip_test_suite"):
+        skip_path = os.path.join(self.download_path, "skip_test_suite")
+        if os.path.exists(skip_path):
             config_f['autospec']['skip_tests'] = 'true'
-            os.remove("skip_test_suite")
+            os.remove(skip_path)
         self.write_config(config_f)
 
     def create_buildreq_cache(self, version, buildreqs_cache):
@@ -479,7 +490,7 @@ class Config(object):
         config_f = configparser.ConfigParser(interpolation=None)
         config_f.read(opts_path)
         if "autospec" not in config_f.sections():
-            print("Missing autospec section in options.conf")
+            print_fatal("Missing autospec section in options.conf")
             sys.exit(1)
 
         if 'package' in config_f.sections() and config_f['package'].get('alias'):
@@ -701,7 +712,7 @@ class Config(object):
             config.read(self.config_file)
 
             if "autospec" not in config.sections():
-                print("Missing autospec section..")
+                print_fatal("Missing autospec section..")
                 sys.exit(1)
 
             self.git_uri = config['autospec'].get('git', None)
@@ -984,6 +995,11 @@ class Config(object):
                     if not license.add_license(word, self.license_translations, self.license_blacklist):
                         print_warning("{}: blacklisted license {} ignored.".format(self.content.name + ".license", word))
 
+        # cargo_vendors is the output of 'cargo vendor' and should be read as is
+        content = self.read_file(os.path.join(self.download_path, "cargo_vendors"), track=True)
+        if content:
+            self.cargo_vendors = "".join(content)
+
         if self.config_opts['use_clang']:
             self.config_opts['funroll-loops'] = False
             requirements.add_buildreq("llvm")
@@ -1002,6 +1018,7 @@ class Config(object):
             requirements.add_buildreq("openssh")
 
         self.prep_prepend = self.read_script_file(os.path.join(self.download_path, "prep_prepend"))
+        self.copy_prepend = self.read_script_file(os.path.join(self.download_path, "copy_prepend"))
         if os.path.isfile(os.path.join(self.download_path, "prep_append")):
             os.rename(os.path.join(self.download_path, "prep_append"), os.path.join(self.download_path, "build_prepend"))
         self.make_prepend = self.read_script_file(os.path.join(self.download_path, "make_prepend"))

autospec/license.py

@@ -178,7 +178,7 @@ def scan_for_licenses(srcdir, config, pkg_name):
             # seen in the DPDK 20.11.3 tree, where the `LICENSES` directory is
             # named `license` instead.
             dirbase = os.path.basename(dirpath)
-            if re.search(r'^(LICENSES|licenses?)$', dirbase) and re.search(r'\.txt$', name):
+            if re.search(r'^(LICENSES|licenses?|licensing)$', dirbase) and re.search(r'\.txt$', name):
                 license_from_copying_hash(os.path.join(dirpath, name),
                                           srcdir, config, pkg_name)
 

autospec/pkg_integrity.py

@@ -42,7 +42,7 @@ KEYID_TRY = ""
 KEYID = ""
 IMPORTED = ""
 EMAIL = ""
-GNUPGCONF = """keyserver keys.gnupg.net"""
+GNUPGCONF = """keyserver keyserver.ubuntu.com"""
 CMD_TIMEOUT = 20
 ENV = os.environ
 INPUT_GETTER_TIMEOUT = 60
@@ -205,9 +205,9 @@ class Verifier(object):
     @staticmethod
     def quit():
         """Stop verification."""
-        print('Critical error quitting...')
+        util.print_fatal("Verification required for build (verify_required option set)")
         print(SEPT)
-        exit(1)
+        sys.exit(1)
 
     @staticmethod
     def calc_sum(filepath, digest_algo):
@@ -264,12 +264,12 @@ def get_signature_file(package_url, package_path):
 def compare_keys(newkey, oldkey):
     """Key comparison to check against key tampering."""
     if newkey != oldkey:
-        util.print_error('Public key has changed:\n'
+        util.print_fatal('Public key has changed:\n'
                          '            old key: {}\n'
                          '            new key: {}\n'
                          'this is a critical security error, quitting...'
                          .format(oldkey, newkey))
-        exit(1)
+        sys.exit(1)
 
 
 # sha256sum Verifier
@@ -521,7 +521,6 @@ class GPGVerifier(Verifier):
 
 def quit_verify():
     """Halt build due to verification being required."""
-    util.print_error("Verification required for build (verify_required option set)")
     Verifier.quit()
 
 
@@ -532,6 +531,7 @@ VERIFIER_TYPES = {
     '.bz2': GPGVerifier,
     '.xz': GPGVerifier,
     '.zip': GPGVerifier,
+    '.zst': GPGVerifier,
 }
 
 

autospec/pypidata.py

@@ -110,7 +110,7 @@ def main():
     pkg_name = sys.argv[1]
     pypi_name = get_pypi_name(pkg_name)
     if not pypi_name:
-        print(f"Couldn't find {pkg_name} in pypi")
+        util.print_fatal(f"Couldn't find {pkg_name} in pypi")
         sys.exit(1)
     pypi_metadata = get_pypi_metadata(pypi_name)
     print(pypi_metadata)

autospec/specfiles.py

@@ -26,6 +26,8 @@ import types
 from collections import OrderedDict
 
 import git
+from jinja2 import Environment
+from jinja2.loaders import DictLoader
 from util import _file_write, open_auto
 
 AVX2_CFLAGS = "-march=x86-64-v3"
@@ -73,12 +75,42 @@ class Specfile(object):
 
     def write_spec(self):
         """Write spec file."""
-        self.specfile = open_auto("{}/{}.spec".format(self.config.download_path, self.name), "w")
+        spec_path = f"{os.path.join(self.config.download_path, self.name)}.spec"
+        self.specfile = open_auto(spec_path, "w")
         self.specfile.write_strip = types.MethodType(_file_write, self.specfile)
 
+        # last chance to sanitize url for template and build types
+        if self.config.urlban:
+            clean_url = re.sub(self.config.urlban, "localhost", self.url)
+            # Duplicate prefixes entry before we change the url
+            self.content.prefixes[clean_url] = self.content.prefixes.get(self.url)
+            self.url = clean_url
+
+        template_path = f"{spec_path}.template"
+
+        if os.path.isfile(template_path):
+            # make templates have a template build pattern
+            self.config.default_pattern = "template"
         # spec file comment header
         self.write_comment_header()
 
+        if os.path.isfile(template_path):
+            with open_auto(template_path) as tfile:
+                template_content = tfile.read()
+            template = Environment(loader=DictLoader({'spec': template_content})).get_template('spec')
+            kw = {
+                'package_name': self.name,
+                'package_version': self.version,
+                'package_url': self.url,
+                'package_release': self.release,
+            }
+            self.specfile.write(template.render(**kw))
+            self.specfile.write_strip('\n')
+            self.specfile.close()
+            # return specfile type built so autospec knows how to
+            # handle build results (template should only builds once)
+            return "template"
+
         if self.config.config_opts.get('keepstatic'):
             self._write("%define keepstatic 1\n")
 
@@ -110,6 +142,10 @@ class Specfile(object):
 
         self.specfile.close()
 
+        # return specfile type built so autospec knows how to
+        # handle build results (generate has multiple builds)
+        return "generate"
+
     def write_comment_header(self):
         """Write comment header to spec file."""
         self._write("#\n")
@@ -132,11 +168,6 @@ class Specfile(object):
 
     def write_nvr(self):
         """Write name, version, and release information."""
-        if self.config.urlban:
-            clean_url = re.sub(self.config.urlban, "localhost", self.url)
-            # Duplicate prefixes entry before we change the url
-            self.content.prefixes[clean_url] = self.content.prefixes.get(self.url)
-            self.url = clean_url
         self._write("Name     : {}\n".format(self.name))
         self._write("Version  : {}\n".format(self.version))
         self._write("Release  : {}\n".format(str(self.release)))
@@ -436,7 +467,7 @@ class Specfile(object):
             cmake_type = "Ninja"
         else:
             cmake_type = "Unix Makefiles"
-        cmake_string = f"cmake -G {cmake_type} -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
+        cmake_string = f"cmake -G '{cmake_type}' -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
                        '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \
                        '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \
                        '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n'
@@ -452,12 +483,16 @@ class Specfile(object):
             self._write_strip(f"gpg --homedir .gnupg --status-fd 1 --verify {self.config.signature_macro} %{{SOURCE0}} > gpg.status")
             self._write_strip(f"grep -E '^\\[GNUPG:\\] (GOODSIG|EXPKEYSIG) {self.keyid}' gpg.status")
         self.write_prep_prepend()
-        prefix = self.content.prefixes[self.url]
         if self.config.default_pattern == 'R':
             prefix = self.content.tarball_prefix
             self._write_strip("%setup -q -n " + prefix)
         else:
-            self._write_strip("%setup -q -n " + prefix)
+            prefix = self.content.prefixes[self.url]
+            if not prefix:
+                prefix = os.path.splitext(os.path.basename(self.url))[0]
+                self._write_strip("%setup -q -c -n " + prefix)
+            else:
+                self._write_strip("%setup -q -n " + prefix)
             for archive in self.config.sources["archive"]:
                 # Handle various archive types
                 extract_cmd = 'tar xf {}'
@@ -465,6 +500,11 @@ class Specfile(object):
                     extract_cmd = 'unzip -q {}'
                 if archive.endswith('.bz2') and not archive.endswith('.tar.bz2'):
                     extract_cmd = 'bzcat {0} > $(basename "{0}" .bz2)'
+                if archive.endswith('.zst'):
+                    if archive.endswith('.tar.zst'):
+                        extract_cmd = 'tar -I zstd xf {}'
+                    else:
+                        extract_cmd = 'zstd -dqc {0} > $(basename "{0}" .zst)'
                 self._write_strip('cd %{_builddir}')
                 archive_file = os.path.basename(archive)
                 if self.config.archive_details.get(archive + "prefix"):
@@ -504,38 +544,36 @@ class Specfile(object):
         self.apply_patches()
 
         # setup cargo.toml vendoring if needed
-        if self.config.config_opts['cargo_vendor']:
+        if self.config.cargo_vendors:
             if self.config.subdir:
                 self._write_strip("pushd " + self.config.subdir)
             self._write_strip("mkdir -p .cargo")
-            self._write_strip("echo '[source.crates-io]' >> .cargo/config.toml")
-            self._write_strip("""echo 'replace-with = "vendored-sources"' >> .cargo/config.toml""")
-            self._write_strip("echo '[source.vendored-sources]' >> .cargo/config.toml")
-            self._write_strip("""echo 'directory = "vendor"' >> .cargo/config.toml""")
+            self._write_strip(f"echo '{self.config.cargo_vendors}' >> .cargo/config.toml")
             if self.config.subdir:
                 self._write_strip("popd")
 
-        if self.config.default_pattern != 'cmake':
-            if self.config.config_opts['32bit']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_avx2']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_avx512']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_apx']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['openmpi']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
-                self._write_strip("popd")
+        self.write_copy_prepend()
+
+        if self.config.config_opts['32bit']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_apx']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
+            self._write_strip("popd")
         self._write_strip("\n")
 
     def write_32bit_exports(self):
@@ -795,6 +833,14 @@ class Specfile(object):
                 self._write_strip("{}\n".format(line))
             self._write_strip("## prep_prepend end")
 
+    def write_copy_prepend(self):
+        """Write out any custom supplied commands prior to creating source copies for avx, etc builds."""
+        if self.config.copy_prepend:
+            self._write_strip("## copy_prepend content")
+            for line in self.config.copy_prepend:
+                self._write_strip("{}\n".format(line))
+            self._write_strip("## copy_prepend end")
+
     def write_build_prepend(self):
         """Write out any custom supplied commands at the start of the %build section and every build type."""
         if self.config.build_prepend:
@@ -910,10 +956,11 @@ class Specfile(object):
 
         self._write_strip("export GOAMD64=v2")
 
-        if self.config.subdir:
-            self._write_strip("pushd " + self.config.subdir)
+        if self.config.config_opts['use_ninja'] and self.config.install_macro == '%make_install':
+            self.config.install_macro = '%ninja_install'
 
         if self.config.config_opts['32bit']:
+            self._write_strip("pushd ../build32/" + self.config.subdir)
             self._write_strip("pushd clr-build32")
             self._write_strip("{}32 {} {}".format(self.config.install_macro,
                                                   self.config.extra_make_install,
@@ -931,30 +978,42 @@ class Specfile(object):
             self._write_strip("    popd")
             self._write_strip("fi")
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-avx2")
             self._write_strip("%s_v3 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
             self._write_strip("GOAMD64=v4")
             self._write_strip("pushd clr-build-avx512")
             self._write_strip("%s_v4 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_apx']:
+            self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-apx")
             self._write_strip("%s_va %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-openmpi")
             self.write_install_openmpi()
             self._write_strip("popd")
+            self._write_strip("popd")
+
+        if self.config.subdir:
+            self._write_strip("pushd " + self.config.subdir)
 
         self._write_strip("GOAMD64=v2")
         self._write_strip("pushd clr-build")
@@ -1648,7 +1707,11 @@ class Specfile(object):
         self.write_make_line()
         self._write_strip("popd")
 
+        if self.config.subdir:
+            self._write_strip("popd")
+
         if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-avx2")
             self._write_strip("pushd clr-build-avx2")
             self.write_build_prepend()
@@ -1661,8 +1724,10 @@ class Specfile(object):
             self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-avx512")
             self._write_strip("pushd clr-build-avx512")
             self.write_build_prepend()
@@ -1675,8 +1740,10 @@ class Specfile(object):
             self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
+            self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-apx")
             self._write_strip("pushd clr-build-apx")
             self.write_build_prepend()
@@ -1689,8 +1756,10 @@ class Specfile(object):
             self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['32bit']:
+            self._write_strip("pushd ../build32/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build32")
             self._write_strip("pushd clr-build32")
             self.write_build_prepend()
@@ -1703,8 +1772,10 @@ class Specfile(object):
             self.write_make_line()
             self._write_strip("unset PKG_CONFIG_PATH")
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-openmpi")
             self._write_strip("pushd clr-build-openmpi")
             self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh")
@@ -1720,8 +1791,6 @@ class Specfile(object):
             self.write_make_line()
             self._write_strip("module unload openmpi")
             self._write_strip("popd")
-
-        if self.config.subdir:
             self._write_strip("popd")
 
         self._write_strip("\n")

autospec/tarball.py

@@ -26,6 +26,7 @@ import tarfile
 import zipfile
 
 import download
+import zstandard as zstd
 from util import do_regex, get_sha1sum, print_fatal, write_out
 
 
@@ -53,6 +54,8 @@ class Source():
             self.type = 'zip'
         elif self.url.lower().endswith(('.bz2')) and not self.url.lower().endswith(('.tar.bz2')):
             self.type = 'bz2'
+        elif self.url.lower().endswith('.zst'):
+            self.type = 'zst'
         else:
             self.type = 'tar'
 
@@ -81,6 +84,16 @@ class Source():
             print_fatal("Not a valid tar file.")
             sys.exit(1)
 
+    def set_zst_prefix(self):
+        """Determine prefix folder name of tar.zst file."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            lines = content.getnames()
+            if len(lines) == 0:
+                print_fatal("Zstd compressed tar file doesn't appear to have any content")
+                sys.exit(1)
+            elif len(lines) > 1:
+                self.prefix = os.path.commonpath(lines)
+
     def set_bz2_prefix(self):
         """No prefix for plain bz2 archives."""
 
@@ -128,6 +141,11 @@ class Source():
         with zipfile.ZipFile(self.path, 'r') as content:
             content.extractall(path=extraction_path)
 
+    def extract_zst(self, extraction_path):
+        """Extract zst in path."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            content.extractall(path=extraction_path)
+
 
 def convert_version(ver_str, name):
     """Remove disallowed characters from the version."""
@@ -277,7 +295,8 @@ class Content():
         if "github.com" in self.url:
             # define regex accepted for valid packages, important for specific
             # patterns to come before general ones
-            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
+            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/.*/(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",

autospec/util.py

@@ -27,6 +27,101 @@ import sys
 dictionary_filename = os.path.dirname(__file__) + "/translate.dic"
 dictionary = [line.strip() for line in open(dictionary_filename, 'r')]
 os_paths = None
+ERROR_FILE = 'pumpAutospec'
+ERROR_ENV = 'AUTOSPEC_UPDATE'
+
+
+def _log_error(error):
+    write_out(ERROR_FILE, f"{error}\n", mode='a')
+
+
+def _commit_result():
+    if not os.path.isfile(ERROR_FILE):
+        return
+    call(f"git add {ERROR_FILE}", check=False, stderr=subprocess.DEVNULL)
+    call(f"git commit {ERROR_FILE} -m 'Notes update'", check=False, stderr=subprocess.DEVNULL)
+    call("git push", check=False, stderr=subprocess.DEVNULL)
+
+
+def _process_line(line, prev_line, current_patch, reported_patches, error):
+    if m := re.match('^Patch #[0-9]+ .(?P<patch>.*).:', line):
+        current_patch[0] = m.group('patch')
+
+    if m := re.match('Hunk #[0-9]+ FAILED at [0-9]+', line):
+        if current_patch[0] not in reported_patches:
+            _log_error("Patch " + current_patch[0] + " does not apply")
+            reported_patches[current_patch[0]] = True
+            return True
+
+    if m := re.match(".*can't find file to patch at input line ", line):
+        if current_patch[0] not in reported_patches:
+            _log_error("Patch " + current_patch[0] + " does not apply")
+            reported_patches[current_patch[0]] = True
+            return True
+
+    if m := re.match('.*meson.build:[0-9]+:[0-9]+: ERROR: Unknown options: "(?P<option>.*)"', line):
+        _log_error("Unknown meson option: " + m.group('option'))
+        return True
+
+    if m := re.match('Error: package ‘(?P<module>.*)’ .* was found, but', line):
+        _log_error("R package " + m.group('module') + " not found")
+        return True
+
+    if m := re.match('.*CMake Error at .*/CMake', prev_line):
+        if m := re.match('(?P<module>.*) not found', line):
+            _log_error("CMake module " + m.group('module') + "not found")
+            return True
+
+    if m := re.match(r'go: download.*connect: connection refused', line):
+        _log_error("Go online update")
+        return True
+
+    if 'Updating crates.io index' in line:
+        _log_error("Rust crates.io online update")
+        return True
+
+    if "error: '__builtin_ctzs' needs isa option -mbmi" in line:
+        _log_error(" error: '__builtin_ctzs' needs isa option -mbmi")
+        return True
+
+    if "error:" in line and 'Bad exit status from' not in line:
+        m = re.match('.*error:(?P<error>.*)', line)
+        if m and not error:
+            _log_error("Compiler: " + m.group('error'))
+            return True
+
+    if m := re.match(r'Could NOT find (?P<package>.*) .missing', line):
+        _log_error("CMake module " + m.group('package') + " not found")
+        return True
+    if m := re.match(r'Could not find a package configuration file provided by (?P<package>.*) with', line):
+        _log_error("CMake module " + m.group('package') + " not found")
+        return True
+    # Unable to find program 'gperf'
+    if m := re.match(r"Failed to find program ‘(?P<module>.*)’", line):
+        _log_error("Failed to find " + m.group('module'))
+        return True
+    if m := re.match(r"Failed to find ‘(?P<module>.*)’", line):
+        _log_error("Failed to find " + m.group('module'))
+        return True
+
+    return False
+
+
+def _process_build_log(filename):
+    with open_auto(filename, "r") as lfile:
+        lines = lfile.readlines()
+
+    prev_line = ''
+    current_patch = ['']
+    reported_patches = {}
+    error = False
+    for line in lines:
+        if _process_line(line, prev_line, current_patch, reported_patches, error):
+            error = True
+        prev_line = line
+
+    if error:
+        _commit_result()
 
 
 def call(command, logfile=None, check=True, **kwargs):
@@ -116,9 +211,22 @@ def print_error(message):
     _print_message(message, 'ERROR', 'red')
 
 
+def print_build_failed():
+    """Print final fatal error, color coded for TTYs."""
+    _print_message('Build failed, aborting', 'FATAL', 'red')
+    try:
+        if os.environ.get(ERROR_ENV):
+            _process_build_log('results/build.log')
+    except Exception:
+        pass
+
+
 def print_fatal(message):
     """Print fatal error, color coded for TTYs."""
     _print_message(message, 'FATAL', 'red')
+    if os.environ.get(ERROR_ENV):
+        write_out(ERROR_FILE, f"{message}\n", mode='a')
+        _commit_result()
 
 
 def print_warning(message):

tests/packageurls

@@ -1612,3 +1612,4 @@ http://sourceforge.net/projects/zsh/files/zsh/5.4.2/zsh-5.4.2.tar.gz,zsh,5.4.2
 https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3.11-pigeonhole-0.5.11.tar.gz,pigeonhole,0.5.11
 https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3-pigeonhole-0.5.20.tar.gz,pigeonhole,0.5.20
 https://www.ezix.org/software/files/lshw-B.02.19.2.tar.gz,lshw,02.19.2
+https://github.com/VectorCamp/vectorscan/archive/refs/tags/vectorscan/5.4.11.tar.gz,vectorscan,5.4.11

tests/test_build.py

@@ -190,25 +190,36 @@ class TestBuildpattern(unittest.TestCase):
         def mock_util_call(cmd):
             del cmd
 
+        def mock_wrap_fatal():
+            output = ['']
+            def mock_print_fatal(msg):
+                output[0] = msg
+            return output, mock_print_fatal
+
+        result, mock_print_fatal = mock_wrap_fatal()
+
+        exit_backup = build.sys.exit
         call_backup = build.util.call
+        print_backup = build.util.print_fatal
+        build.sys.exit = MagicMock()
         build.util.call = mock_util_call
+        build.util.print_fatal = mock_print_fatal
 
         open_name = 'build.util.open_auto'
         content = "line1\nDEBUG util.py:399:  No matching package to install: 'foobar'\nDEBUG util.py:399:  No matching package to install: 'foobarbaz'\nline 4"
         m_open = mock_open(read_data=content)
         pkg = build.Build()
-        pkg.must_restart = 1
-        pkg.file_restart = 1
 
-        result = True
         with patch(open_name, m_open, create=True):
-            result = pkg.parse_buildroot_log('testname', 1)
+            pkg.parse_buildroot_log('testname', 1)
 
+        build.util.print_fatal = print_backup
         build.util.call = call_backup
+        mock_exit = build.sys.exit
+        build.sys.exit = exit_backup
 
-        self.assertFalse(result)
-        self.assertEqual(pkg.must_restart, 0)
-        self.assertEqual(pkg.file_restart, 0)
+        self.assertEqual(result[0], 'Cannot resolve dependency name: foobar\nCannot resolve dependency name: foobarbaz')
+        mock_exit.assert_called_once()
 
     def test_parse_buildroot_log_pass(self):
         """
@@ -227,11 +238,10 @@ class TestBuildpattern(unittest.TestCase):
 
         result = True
         with patch(open_name, m_open, create=True):
-            result = pkg.parse_buildroot_log('testname', 1)
+            pkg.parse_buildroot_log('testname', 1)
 
         build.util.call = call_backup
 
-        self.assertTrue(result)
         self.assertEqual(pkg.must_restart, 0)
 
     def test_parse_buildroot_log_noop(self):

tests/test_specfile.py

@@ -47,7 +47,7 @@ class TestSpecfileWrite(unittest.TestCase):
         self.WRITES = self.WRITES[:4] + self.WRITES[6:]
         self.assertEqual(expect, self.WRITES)
 
-    def test_write_nvr_no_urlban(self):
+    def test_write_nvr(self):
         """
         test Specfile.write_nvr with no urlban set
         """
@@ -59,19 +59,6 @@ class TestSpecfileWrite(unittest.TestCase):
                   "Source0  : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n"]
         self.assertEqual(expect, self.WRITES)
 
-    def test_write_nvr_urlban(self):
-        """
-        test Specfile.write_nvr with urlban set
-        """
-        self.specfile.config.urlban = "www.testpkg.com"
-        self.specfile.write_nvr()
-        expect = ["Name     : pkg\n",
-                  "Version  : 1.0\n",
-                  "Release  : 2\n",
-                  "URL      : http://localhost/testpkg/pkg-1.0.tar.gz\n",
-                  "Source0  : http://localhost/testpkg/pkg-1.0.tar.gz\n"]
-        self.assertEqual(expect, self.WRITES)
-
     def test_write_sources(self):
         """
         test write_sources with all Specfile.sources set.