Limit diff to ~10 lines

Also include content of added files.

README.rst

@@ -128,14 +128,6 @@ release
   ``.spec``. This is also bumped and generated on existing and new packages,
   respectively. This results in less manual work via automatic management.
 
-license_skips
-  Each line in the file should be the path to a license file. That path needs
-  to account for the package tarfile prefix. Likely requires using '*' to be
-  effective (e.g. ``pkgname-*/path/to/license`` where ``*`` handles the version).
-
-  Files paths can contain a single '*' per directory such that
-  a line of ``/foo*/bar*`` is allowed but ``/foo*bar*`` is not.
-
 $package.license
   In certain cases, the package license may not be automatically discovered.  In
   this instance, ``autospec`` will exit with an error. Update this file to
@@ -212,12 +204,6 @@ ${custom}_provides_ban
 Controlling the build process
 ------------------------------
 
-invalid_release_sig
-  This file contains the current version that will **not** have its package
-  file be processed for signature verification (overriding the config_opt).
-  This file will be automatically deleted after a new release and is intended
-  to override a single bad signed release.
-
 extra_sources
   This file contains a list of extra files to be added to the ``.spec`` and
   optionally installed as well. Each non-blank and non-comment line should start
@@ -282,12 +268,6 @@ prep_prepend
   resulting ``.spec``, and is used for situations where fine-grained
   control is required.
 
-copy_prepend
-  Additional actions that should take place directly before the source
-  directory is copied for other builds (32bit, avx2, etc). This will be
-  placed in the resulting ``.spec``, and is used for situations where
-  fine-grained control is required.
-
 build_prepend
   Additional actions that should take place after ``%build`` and before
   the ``%configure`` macro or equivalent (``%cmake``, etc.). If autospec

autospec/abireport.py

@@ -155,7 +155,7 @@ def is_file_valid(path):
 
 def dump_symbols(path):
     """Get symbols from a file."""
-    cmd = "nm --defined-only -g --dynamic \"{}\" | c++filt".format(path)
+    cmd = "nm --defined-only -g --dynamic \"{}\"".format(path)
     lines = None
 
     ret = set()
@@ -163,7 +163,7 @@ def dump_symbols(path):
     try:
         lines = get_output(cmd)
     except Exception as e:
-        util.print_fatal("Fatal error inspecting {}: {}".format(path, e))
+        print("Fatal error inspecting {}: {}".format(path, e))
         sys.exit(1)
     for line in lines.split("\n"):
         line = line.strip()
@@ -236,7 +236,6 @@ def examine_abi_host(download_path, results_dir, name):
                   cwd=download_path)
     except Exception as e:
         util.print_fatal("Error invoking abireport: {}".format(e))
-        sys.exit(1)
 
 
 def examine_abi_fallback(download_path, results_dir, name):
@@ -273,7 +272,6 @@ def examine_abi_fallback(download_path, results_dir, name):
             subprocess.check_call(cmd, shell=True)
     except Exception as e:
         util.print_fatal("Error extracting RPMS: {}".format(e))
-        sys.exit(1)
 
     os.chdir(download_path)
     collected_files = set()

autospec/autospec.py

@@ -38,7 +38,7 @@ import specfiles
 import tarball
 from abireport import examine_abi
 from logcheck import logcheck
-from util import binary_in_path, print_build_failed, print_fatal, write_out
+from util import binary_in_path, print_fatal, write_out
 
 sys.path.append(os.path.dirname(__file__))
 
@@ -269,13 +269,9 @@ def package(args, url, name, archives, workingdir):
         pkg_integrity.check(url, conf, interactive=interactive_mode)
         pkg_integrity.load_specfile(specfile)
 
-    spec_type = specfile.write_spec()
-
+    specfile.write_spec()
     while 1:
         package.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, content, args.cleanup)
-        if spec_type == "template":
-            # specfile template is assumed "correct" and any failures need to be manually addressed
-            break
         filemanager.load_specfile(specfile)
         specfile.write_spec()
         filemanager.newfiles_printed = 0
@@ -295,7 +291,7 @@ def package(args, url, name, archives, workingdir):
 
     if package.success == 0:
         conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
-        print_build_failed()
+        print_fatal("Build failed, aborting")
         sys.exit(1)
     elif os.path.isfile("README.clear"):
         try:
@@ -308,8 +304,7 @@ def package(args, url, name, archives, workingdir):
         except Exception:
             pass
 
-    if spec_type == "generate":
-        check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
+    check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
 
     examine_abi(conf.download_path, content.name)
     if os.path.exists("/var/lib/rpm"):

autospec/build.py

@@ -22,7 +22,6 @@
 import os
 import re
 import shutil
-import sys
 
 import util
 
@@ -154,7 +153,7 @@ class Build(object):
             return True
         self.must_restart = 0
         self.file_restart = 0
-        fatals = []
+        is_clean = True
         util.call("sync")
         with util.open_auto(filename, "r") as rootlog:
             loglines = rootlog.readlines()
@@ -162,10 +161,9 @@ class Build(object):
         for line in loglines:
             match = missing_pat.match(line)
             if match is not None:
-                fatals.append(f"Cannot resolve dependency name: {match.group(1)}")
-        if fatals:
-            util.print_fatal('\n'.join(fatals))
-            sys.exit(1)
+                util.print_fatal("Cannot resolve dependency name: {}".format(match.group(1)))
+                is_clean = False
+        return is_clean
 
     def parse_build_results(self, filename, returncode, filemanager, config, requirements, content):
         """Handle build log contents."""
@@ -285,7 +283,7 @@ class Build(object):
             mockopts,
         ]
 
-        if config.config_opts.get('avoid_rebuild') and not cleanup and self.must_restart == 0 and self.file_restart > 0 and set(filemanager.excludes) == set(filemanager.manual_excludes):
+        if not cleanup and self.must_restart == 0 and self.file_restart > 0 and set(filemanager.excludes) == set(filemanager.manual_excludes):
             cmd_args.append("--no-clean")
             cmd_args.append("--short-circuit=binary")
 
@@ -297,11 +295,12 @@ class Build(object):
         # sanity check the build log
         if not os.path.exists(config.download_path + "/results/build.log"):
             util.print_fatal("Mock command failed, results log does not exist. User may not have correct permissions.")
-            sys.exit(1)
+            exit(1)
 
-        self.parse_buildroot_log(config.download_path + "/results/root.log", ret)
+        if not self.parse_buildroot_log(config.download_path + "/results/root.log", ret):
+            return
 
         self.parse_build_results(config.download_path + "/results/build.log", ret, filemanager, config, requirements, content)
         if filemanager.has_banned:
             util.print_fatal("Content in banned paths found, aborting build")
-            sys.exit(1)
+            exit(1)

autospec/buildreq.py

@@ -147,7 +147,7 @@ def parse_modules_list(modules_string, is_cmake=False):
         modules = [m for m in re.split(r'\s*([><]?=|\${?[^}]*}?)\s*', modules_string)]
         modules = filter(None, modules)
     else:
-        modules = [m.strip('[]').strip('"') for m in modules_string.split()]
+        modules = [m.strip('[]') for m in modules_string.split()]
     res = []
     next_is_ver = False
     for mod in modules:
@@ -508,7 +508,7 @@ class Requirements(object):
         """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for."""
         findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I)
         findpackage_multiline = re.compile(r"^[^#]*find_package\((\w+)\b.*", re.I)
-        pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\([\w\-]+ (.*)\)", re.I)
+        pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I)
         pkg_search_modifiers = {'REQUIRED', 'QUIET', 'NO_CMAKE_PATH',
                                 'NO_CMAKE_ENVIRONMENT_PATH', 'IMPORTED_TARGET'}
         extractword = re.compile(r'(?:"([^"]+)"|(\S+))(.*)')
@@ -518,13 +518,8 @@ class Requirements(object):
         for line in lines:
             if match := findpackage.search(line):
                 module = match.group(1)
-                if pkgs := cmake_modules.get(module):
-                    # Some of the entries in cmake_modules list multiple packages, space-separated, so we need to split.
-                    # Otherwise, anything in buildreq_ban would have to match the entire string, not just a single package name.
-                    # For example: Png2Ico, extra-cmake-modules png2ico
-                    # buildreq_ban would have to contain "extra-cmake-modules png2ico" to match, instead of just "png2ico"
-                    for pkg in pkgs.split():
-                        self.add_buildreq(pkg)
+                if pkg := cmake_modules.get(module):
+                    self.add_buildreq(pkg)
             elif findpackage_multiline.search(line):
                 self.findpackage_parse_lines(line, lines, cmake_modules)
 
@@ -756,14 +751,10 @@ class Requirements(object):
         configure_ac_files = []
         qmake_profiles = []
         cmake_files = []
-
-        if config.config_opts['use_ninja']:
-            self.add_buildreq('ninja')
-
         for dirpath, _, files in os.walk(dirn):
             default_score = 2 if dirpath == dirn else 1
 
-            if "Cargo.toml" in files and 'Makefile' not in files:
+            if "Cargo.toml" in files:
                 config.set_build_pattern('cargo', default_score)
 
             if "CMakeLists.txt" in files and "configure.ac" not in files:

autospec/check.py

@@ -64,54 +64,50 @@ def scan_for_tests(src_dir, config, requirements, content):
     if config.config_opts.get('skip_tests') or tests_config:
         return
 
-    make_command = "ninja" if config.config_opts.get('use_ninja') else "make"
     makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
-    make_check = "{} {}check".format(make_command, makeflags)
-    cmake_check = "{} test".format(make_command)
+    make_check = "make {}check".format(makeflags)
+    cmake_check = "make test"
     make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
-                         "{} {}check\nmodule unload openmpi".format(make_command, makeflags)
+                         "make {}check\nmodule unload openmpi".format(makeflags)
     cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
-                          "{} test\nmodule unload openmpi".format(make_command)
+                          "make test\nmodule unload openmpi"
 
     if config.config_opts.get('allow_test_failures'):
         make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
-                             "{} {}check || :\nmodule unload openmpi".format(make_command, makeflags)
+                             "make {}check || :\nmodule unload openmpi".format(makeflags)
         cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
-                              "{} test || :\nmodule unload openmpi".format(make_command)
+                              "make test || :\nmodule unload openmpi"
 
-    perl_check = "{} TEST_VERBOSE=1 test".format(make_command)
+    perl_check = "make TEST_VERBOSE=1 test"
+    setup_check = """PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") python setup.py test"""
     meson_check = "meson test -C builddir --print-errorlogs"
     if config.config_opts.get('allow_test_failures'):
         make_check += " || :"
         cmake_check += " || :"
         perl_check += " || :"
+        setup_check += " || :"
         meson_check += " || :"
 
     testsuites = {
         "makecheck": make_check,
         "perlcheck": perl_check,
+        "setup.py": setup_check,
         "cmake": "cd clr-build; " + cmake_check,
         "meson": meson_check,
     }
     if config.config_opts.get('32bit'):
         testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../../build32/clr-build32;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
         testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
     if config.config_opts.get('use_avx2'):
         testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../../buildavx2/clr-build-avx2;\n" + cmake_check + " || :"
-        testsuites["meson"] += "\ncd ../buildavx2;\n" + meson_check + " || :"
+        testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
     if config.config_opts.get('use_avx512'):
         testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../../buildavx512/clr-build-avx512;\n" + cmake_check + " || :"
-        testsuites["meson"] += "\ncd ../buildavx512;\n" + meson_check + " || :"
-    if config.config_opts.get('use_apx'):
-        testsuites["makecheck"] += "\ncd ../buildapx;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../../buildapx/clr-build-apx;\n" + cmake_check + " || :"
-        testsuites["meson"] += "\ncd ../buildapx;\n" + meson_check + " || :"
+        testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
     if config.config_opts.get('openmpi'):
         testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
-        testsuites["cmake"] += "\ncd ../../build-openmpi/clr-build-openmpi;\n" + cmake_check_openmpi
+        testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi
 
     files = os.listdir(src_dir)
 
@@ -142,6 +138,13 @@ def scan_for_tests(src_dir, config, requirements, content):
     elif config.default_pattern in ["cpan"] and "Makefile.PL" in files:
         tests_config = testsuites["perlcheck"]
 
+    elif config.default_pattern == "distutils3" and "setup.py" in files:
+        with util.open_auto(os.path.join(src_dir, "setup.py"), 'r') as setup_fp:
+            setup_contents = setup_fp.read()
+
+        if "test_suite" in setup_contents or "pbr=True" in setup_contents:
+            tests_config = testsuites["setup.py"]
+
     elif config.default_pattern == "R":
         tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n"              \
                        "R CMD check --no-manual --no-examples --no-codoc . "    \

autospec/commitmessage.py

@@ -49,7 +49,7 @@ def scan_for_changes(download_path, directory, transforms):
                 shutil.copy(source, target)
                 os.chmod(target, 0o644)
             except Exception as e:
-                util.print_fatal("Error copying file: {}".format(e))
+                print("Error copying file: {}".format(e))
                 sys.exit(1)
             found.append(item)
 

autospec/config.py

@@ -28,8 +28,7 @@ from collections import OrderedDict
 
 import check
 import license
-from util import (call, open_auto, print_fatal, print_info, print_warning,
-                  write_out)
+from util import call, open_auto, print_info, print_warning, write_out
 
 
 def read_pattern_conf(filename, dest, list_format=False, path=None):
@@ -69,7 +68,6 @@ class Config(object):
 
     def __init__(self, download_path):
         """Initialize Default configuration settings."""
-        self.cargo_vendors = ""
         self.content = None  # hack to avoid circular init dependency
         self.extra_configure = ""
         self.extra_configure32 = ""
@@ -93,7 +91,6 @@ class Config(object):
         self.install_macro = "%make_install"
         self.disable_static = "--disable-static"
         self.prep_prepend = []
-        self.copy_prepend = []
         self.build_prepend = []
         self.build_prepend_once = []
         self.build_append = []
@@ -110,7 +107,6 @@ class Config(object):
         self.custom_summ = ""
         self.license_fetch = None
         self.license_show = None
-        self.license_skips = []
         self.git_uri = None
         self.os_packages = set()
         self.config_file = None
@@ -158,7 +154,6 @@ class Config(object):
         }
         self.config_opts = {}
         self.config_options = {
-            "avoid_rebuild": "Try to use mock short circuit to avoid full rebuilds",
             "broken_c++": "extend flags with '-std=gnu++98",
             "cargo_vendor": "create vendor archive with cargo",
             "use_lto": "configure build for lto",
@@ -192,7 +187,6 @@ class Config(object):
             "server": "Package is only used by servers",
             "no_glob": "Do not use the replacement pattern for file matching",
             "allow_exe": "Allow Windows executables (*.exe, *.dll) to be packaged",
-            "use_ninja": "Use ninja build files",
         }
         # simple_pattern_pkgconfig patterns
         # contains patterns for parsing build.log for missing dependencies
@@ -272,7 +266,6 @@ class Config(object):
             (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC \(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'),
             (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None),
             (r"Checking for (.*?)\.\.\.no", 0, None),
-            (r"checking for (.*?) \(using pkg-config\)\.\.\. no", 0, None),
             (r"Checking for (.*?)\s*: not found", 0, None),
             (r"Checking for (.*?)\s>=.*\s*: not found", 0, None),
             (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None),
@@ -295,12 +288,10 @@ class Config(object):
             (r"No rule to make target `(.*)',", 0, None),
             (r"Package (.*) was not found in the pkg-config search path.", 0, 'pkgconfig'),
             (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'),
-            (r"The file `([a-zA-Z0-9\-:]*)\.pc` needs to be installed and the PKG_CONFIG_PATH environment variable must contain its parent directory\.", 0, 'pkgconfig'),
             (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'),
             (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'),
             (r"Program (.*) found: NO", 0, None),
             (r"Target '[a-zA-Z0-9\-]' can't be generated as '(.*)' could not be found", 0, None),
-            (r"The missing Perl modules are:\s*(\S+)", 0, 'perl'),
             (r"Unable to `import (.*)`", 0, None),
             (r"Unable to find '(.*)'", 0, None),
             (r"Warning: prerequisite ([a-zA-Z:]+) [0-9\.]+ not found.", 0, 'perl'),
@@ -321,7 +312,6 @@ class Config(object):
             (r"checking for [a-zA-Z0-9\_\-]+ in (.*?)\.\.\. no", 0, None),
             (r"checking for library containing (.*)... no", 0, None),
             (r"checking for perl module ([a-zA-Z:]+) [0-9\.]+... no", 0, 'perl'),
-            ("checking if (.*) is available...", 0, None),
             (r"configure: error: (?:pkg-config missing|Unable to locate) (.*)", 0, None),
             (r"configure: error: ([a-zA-Z0-9]+) (?:is required to build|not found)", 0, None),
             (r"configure: error: Cannot find (.*)\. Make sure", 0, None),
@@ -347,7 +337,7 @@ class Config(object):
     def detect_build_from_url(self, url):
         """Detect build patterns and build requirements from the patterns detected in the url."""
         # R package
-        if "cran.r-project.org" in url or "cran.rstudio.com" in url or "/pub/cran/" in url:
+        if "cran.r-project.org" in url or "cran.rstudio.com" in url:
             self.set_build_pattern("R", 10)
 
         # python
@@ -444,24 +434,20 @@ class Config(object):
         # next the options
         config_f['autospec'] = {}
         for fname, comment in sorted(self.config_options.items()):
-            fpath = os.path.join(self.download_path, fname)
             config_f.set('autospec', '# {}'.format(comment))
-            if os.path.isfile(fpath):
+            if os.path.exists(fname):
                 config_f['autospec'][fname] = 'true'
-                os.remove(fpath)
+                os.remove(fname)
             else:
                 config_f['autospec'][fname] = 'false'
 
         # default lto to true for new things
         config_f['autospec']['use_lto'] = 'true'
-        # default autoupdate to true for new things
-        config_f['autospec']['autoupdate'] = 'true'
 
         # renamed options need special care
-        skip_path = os.path.join(self.download_path, "skip_test_suite")
-        if os.path.exists(skip_path):
+        if os.path.exists("skip_test_suite"):
             config_f['autospec']['skip_tests'] = 'true'
-            os.remove(skip_path)
+            os.remove("skip_test_suite")
         self.write_config(config_f)
 
     def create_buildreq_cache(self, version, buildreqs_cache):
@@ -492,7 +478,7 @@ class Config(object):
         config_f = configparser.ConfigParser(interpolation=None)
         config_f.read(opts_path)
         if "autospec" not in config_f.sections():
-            print_fatal("Missing autospec section in options.conf")
+            print("Missing autospec section in options.conf")
             sys.exit(1)
 
         if 'package' in config_f.sections() and config_f['package'].get('alias'):
@@ -714,7 +700,7 @@ class Config(object):
             config.read(self.config_file)
 
             if "autospec" not in config.sections():
-                print_fatal("Missing autospec section..")
+                print("Missing autospec section..")
                 sys.exit(1)
 
             self.git_uri = config['autospec'].get('git', None)
@@ -837,18 +823,6 @@ class Config(object):
             except Exception as e:
                 print_warning(f"Unable to remove buildreq_cache file: {e}")
 
-        invalid_release_sig_file = os.path.join(self.download_path, "invalid_release_sig")
-        content = self.read_conf_file(invalid_release_sig_file)
-        if content and content[0] == version:
-            self.config_opts['verify_required'] = False
-        else:
-            try:
-                os.unlink(invalid_release_sig_file)
-            except FileNotFoundError:
-                pass
-            except Exception as e:
-                print_warning(f"Unable to remove invalid_release_sig file: {e}")
-
         content = self.read_conf_file(os.path.join(self.download_path, "pkgconfig_add"))
         for extra in content:
             extra = pkgconfig_re.sub(r'\1', extra)
@@ -1009,15 +983,6 @@ class Config(object):
                     if not license.add_license(word, self.license_translations, self.license_blacklist):
                         print_warning("{}: blacklisted license {} ignored.".format(self.content.name + ".license", word))
 
-        content = self.read_conf_file(os.path.join(self.download_path, "license_skips"))
-        if content:
-            self.license_skips = self.validate_extras_content(content, "license_skips")
-
-        # cargo_vendors is the output of 'cargo vendor' and should be read as is
-        content = self.read_file(os.path.join(self.download_path, "cargo_vendors"), track=True)
-        if content:
-            self.cargo_vendors = "".join(content)
-
         if self.config_opts['use_clang']:
             self.config_opts['funroll-loops'] = False
             requirements.add_buildreq("llvm")
@@ -1036,7 +1001,6 @@ class Config(object):
             requirements.add_buildreq("openssh")
 
         self.prep_prepend = self.read_script_file(os.path.join(self.download_path, "prep_prepend"))
-        self.copy_prepend = self.read_script_file(os.path.join(self.download_path, "copy_prepend"))
         if os.path.isfile(os.path.join(self.download_path, "prep_append")):
             os.rename(os.path.join(self.download_path, "prep_append"), os.path.join(self.download_path, "build_prepend"))
         self.make_prepend = self.read_script_file(os.path.join(self.download_path, "make_prepend"))

autospec/failed_commands

@@ -124,9 +124,7 @@ ACL_TYPE_EXTENDED, acl-dev
 ALSA, alsa-lib-dev
 APR, apr-dev
 APR-util, apr-util-dev
-ARPACK-NG, arpack-ng-dev
 ASSIMP, assimp-dev
-Adwaita, libadwaita-dev
 Analitza, analitza-dev
 Analitza5, analitza-dev
 BABL, babl-dev
@@ -177,7 +175,6 @@ EXPAT, expat-dev
 Eet.h, eet-dev
 Eigen/Core, eigen-dev
 Eigen/Dense, eigen-dev
-Eigen/Sparse, eigen-dev
 Eigen3, eigen-dev
 Epoxy, pkgconfig(epoxy)
 Exiv2, exiv2-dev
@@ -287,7 +284,6 @@ GSTREAMER, gstreamer-dev
 GSettingSchemas, gsettings-desktop-schemas
 GSettingSchemas, gsettings-desktop-schemas-dev
 GTK, gtk+-dev
-GTK4, gtk4-dev
 GnuTLS, gnutls-dev
 GooglePerftools, gperftools-dev
 Gperf, gperf
@@ -463,7 +459,6 @@ KF6NewStuff, knewstuff-def
 KF6Notifications, knotifications-dev
 KF6NotifyConfig, knotifyconfig-dev
 KF6Package, kpackage-dev
-KF6Prison, kprison-dev
 KF6Purpose, purpose-dev
 KF6QuickCharts, kquickcharts-dev
 KF6Runner, krunner-dev
@@ -553,7 +548,6 @@ NETTLE, nettle-dev
 NUMA, numactl-dev
 Nepomuk, nepomuk-core-dev
 NetworkManager, NetworkManager-dev
-Nice, libnice-dev
 NumPy, pypi-numpy
 Numa, numactl-dev
 OPENEXR, openexr-dev
@@ -627,7 +621,6 @@ Qt5Xdg, libqtxdg-dev
 Qt5XdgIconLoader, libqtxdg-dev
 Qt5XmlPatterns, qtxmlpatterns-dev
 Qt6Positioning, qt6positioning-dev
-Qt6Qml, qt6declarative-dev
 Qt6Quick, qt6declarative-dev
 Qt6QuickTimeline, qt6quicktimeline-dev
 Qt6ShaderTools, qt6shadertools-dev
@@ -657,7 +650,6 @@ Soprano, soprano-dev
 SpatialIndex, libspatialindex-dev
 Sphinx, pypi-sphinx
 Sqlite, sqlite-autoconf-dev
-Srtp2, libsrtp-dev
 Startup notification library, libnotify-dev
 TBB, tbb-dev
 TIFF, tiff-dev
@@ -1132,7 +1124,6 @@ libsecret-unstable, libsecret-dev
 libsoup-2.4, libsoup-dev
 libssh, libssh-dev
 libssh2.h, libssh2-dev
-libssl, openssl-dev
 libsystemd, pkgconfig(systemd)
 libtirpc (via pkg-config), pkgconfig(libtirpc)
 libudev.h, systemd-dev
@@ -1307,8 +1298,6 @@ pycparser, pypi(pycparser)
 pycurl, pypi(pycurl)
 pylint, pypi(pylint)
 pyperclip, pypi(pyperclip)
-pypi(openvino), openvino-python3
-pypi(pytorch), pytorch-python3
 pytest, pypi-pytest
 pytest-cov, pypi(pytest_cov)
 python lxml, pypi(lxml)
@@ -1503,7 +1492,6 @@ yaml-cpp, yaml-cpp-dev
 yarg, pypi-yarg
 yasm, yasm
 yytext is a pointer, flex
-zconf.h, zlib-dev
 zenity, zenity
 zip, zip
 zlib, zlib-dev

autospec/files.py

@@ -197,6 +197,33 @@ class FileManager(object):
 
         return removed
 
+    def globlike_match(self, filename, match_name):
+        """Compare the filename to the match_name in a way that simulates the shell glob '*'."""
+        fsplit = filename.split('/')
+        if len(fsplit) != len(match_name):
+            return False
+        match = True
+        for fpart, mpart in zip(fsplit, match_name):
+            if fpart != mpart:
+                if '*' not in mpart:
+                    match = False
+                    break
+                if len(mpart) > len(fpart) + 1:
+                    match = False
+                    break
+                mpl, mpr = mpart.split('*')
+                try:
+                    if fpart.index(mpl) != 0:
+                        match = False
+                        break
+                    if fpart.rindex(mpr) != len(fpart) - len(mpr):
+                        match = False
+                        break
+                except ValueError:
+                    match = False
+                    break
+        return match
+
     def push_file(self, filename, pkg_name):
         """Perform a number of checks against the filename and push the filename if appropriate."""
         if filename in self.files or filename in self.files_blacklist:
@@ -218,7 +245,7 @@ class FileManager(object):
                 elif len('/'.join(match_name)) <= (len(norm_filename) + 1):
                     # the match name may be 1 longer due to a glob
                     # being able to match an empty string
-                    if util.globlike_match(norm_filename, match_name):
+                    if self.globlike_match(norm_filename, match_name):
                         path_prefix = '/' if not match else match.group()
                         self.push_package_file(os.path.join(path_prefix, *match_name), k)
                         return
@@ -377,7 +404,6 @@ class FileManager(object):
             (r"^/lib/systemd/user/", "services"),
             (r"^/usr/lib/systemd/system/", "services"),
             (r"^/usr/lib/systemd/user/", "services"),
-            (r"^/usr/lib/udev/hwdb.d", "config"),
             (r"^/usr/lib/udev/rules.d", "config"),
             (r"^/usr/lib/modules-load.d", "config"),
             (r"^/usr/lib/tmpfiles.d", "config"),

autospec/git.py

@@ -113,6 +113,7 @@ def commit_to_git(config, name, success):
     call("git add whatrequires", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add description", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add attrs", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add archive.diff", check=False, stderr=subprocess.DEVNULL, cwd=path)
 
     # remove deprecated config files
     call("git rm make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)

autospec/license.py

@@ -29,7 +29,7 @@ import urllib.parse
 
 import chardet
 import download
-import util
+from util import get_contents, get_sha1sum, print_fatal, print_warning
 
 default_license = "TO BE DETERMINED"
 
@@ -96,7 +96,7 @@ def decode_license(license):
 def license_from_copying_hash(copying, srcdir, config, name):
     """Add licenses based on the hash of the copying file."""
     try:
-        data = util.get_contents(copying)
+        data = get_contents(copying)
     except FileNotFoundError:
         # LICENSE file is a bad symlink (qemu-4.2.0!)
         return
@@ -109,7 +109,7 @@ def license_from_copying_hash(copying, srcdir, config, name):
     if not data:
         return
 
-    hash_sum = util.get_sha1sum(copying)
+    hash_sum = get_sha1sum(copying)
 
     if config.license_fetch:
         values = {'hash': hash_sum, 'text': data, 'package': name}
@@ -142,24 +142,9 @@ def license_from_copying_hash(copying, srcdir, config, name):
     else:
         if not config.license_show:
             return
-        util.print_warning("Unknown license {0} with hash {1}".format(copying, hash_sum))
+        print_warning("Unknown license {0} with hash {1}".format(copying, hash_sum))
         hash_url = config.license_show % {'HASH': hash_sum}
-        util.print_warning("Visit {0} to enter".format(hash_url))
-
-
-def skip_license(license_path, config):
-    """Check if a given license file path should be skipped."""
-    skip_name = False
-    for skip in config.license_skips:
-        # handle the common tempfile prefix and normalize for
-        # skip lines without a starting '/'
-        skip = skip if skip[0] != '' else skip[1:]
-        skip_path = ['', 'tmp', '*'] + skip
-        if util.globlike_match(license_path, skip_path):
-            util.print_warning(f"Skip license detected for file at {license_path}")
-            skip_name = True
-            break
-    return skip_name
+        print_warning("Visit {0} to enter".format(hash_url))
 
 
 def scan_for_licenses(srcdir, config, pkg_name):
@@ -181,9 +166,8 @@ def scan_for_licenses(srcdir, config, pkg_name):
     for dirpath, dirnames, files in os.walk(srcdir):
         for name in files:
             if name.lower() in targets or target_pat.search(name.lower()):
-                license_path = os.path.join(dirpath, name)
-                if not skip_license(license_path, config):
-                    license_from_copying_hash(license_path, srcdir, config, pkg_name)
+                license_from_copying_hash(os.path.join(dirpath, name),
+                                          srcdir, config, pkg_name)
             # Also search for license texts in project trees that are
             # REUSE-compliant, or are in process of adopting this standard (for
             # example, KDE ecosystem packages). See https://reuse.software for
@@ -194,13 +178,12 @@ def scan_for_licenses(srcdir, config, pkg_name):
             # seen in the DPDK 20.11.3 tree, where the `LICENSES` directory is
             # named `license` instead.
             dirbase = os.path.basename(dirpath)
-            if re.search(r'^(LICENSES|licenses?|licensing)$', dirbase) and re.search(r'\.txt$', name):
-                license_path = os.path.join(dirpath, name)
-                if not skip_license(license_path, config):
-                    license_from_copying_hash(license_path, srcdir, config, pkg_name)
+            if re.search(r'^(LICENSES|licenses?)$', dirbase) and re.search(r'\.txt$', name):
+                license_from_copying_hash(os.path.join(dirpath, name),
+                                          srcdir, config, pkg_name)
 
     if not licenses:
-        util.print_fatal(" Cannot find any license or a valid {}.license file!\n".format(pkg_name))
+        print_fatal(" Cannot find any license or a valid {}.license file!\n".format(pkg_name))
         sys.exit(1)
 
     print("Licenses    : ", " ".join(sorted(licenses)))

autospec/license_blacklist

@@ -3,12 +3,12 @@
 # This file is sorted with LC_COLLATE=C
 # Lines beginning with '#' are ignored.
 # For strings that start with '#', escape the '#' as '\#'.
+=========================
 %
 %license
 (LGPL)
 (new)
 (specified
-*
 +
 -
 -MIT
@@ -26,8 +26,6 @@
 3-Clause
 3-clause
 3BSD
-=========================
->=-2
 @CPACK_RPM_PACKAGE_LICENSE@
 AGPLv3+
 ALv2
@@ -37,12 +35,7 @@ APPLICATION__TYPE
 ASL
 Artistic
 BSD
-BSD(2
 BSD(3
-BSD-2-Clause-Views
-BSD-3
-BSD-compatible
-BSD-derived(Repoze)
 BSD-like
 BSD-style
 BSDL
@@ -50,7 +43,6 @@ BSD_3_clause
 BSDish
 BSL
 CC-BY
-Clause
 Commercial
 Corp.
 Distribution
@@ -61,7 +53,6 @@ EPL
 Eclipse
 Expat(MIT/X11)
 Expat/MIT
-FOUNDATION
 Foundation
 FreeBSD
 GENERAL
@@ -69,39 +60,30 @@ GFDL
 GNU
 GPL
 GPL+
-GPL-1.0-only
 GPL/BSD
 GPL/BSD/CPL
 GPLv2.1
 General
 IBM
-Jupyter
 LESSER
 LGPL
 LGPL+BSD
 LGPL/MIT
 LICENCE
 Lesser
-Library
 Licences
 License
 License(2.0)
 License(==-2.0)
-License(>=-2)
 License(>=-2.0)
-License(LGPL)
-License(MIT)
 License,
 License-2
 License-2.0
 License-2.0(MPL-2.0)
-License-3(GPLv3)
 Licensing
 Lucent
-MIT-0
 MIT/Expat
 MPL
-Minpack
 Modified
 Mozilla
 Muddy-MIT
@@ -113,13 +95,9 @@ Open
 PIL
 PSF-2+
 PUBLIC
-PYTHON
-Permission
 Public
 Revised
 SIL
-SIP
-SOFTWARE
 See
 Set
 Software
@@ -133,46 +111,31 @@ UNKNOWN
 Unknown
 Unkown
 Unlimited
-VERSION-2
 Version
 Version-2.0
-Version-3
-WITH
 What
 \#
-a
 advertising
 and
 any
 bsd
-charge
 classifiers)
 clause)
-copy
-cryptsetup-OpenSSL-exception
 details
 domain
-domain.
 dual
 exceptions
 for
-free
 gpl
-granted
-hereby
 http://nmap.org/man/man-legal.html
-into
 is
 it
 later
 later(LGPLv2+)
 license
-licensing
 ndg/httpsclient/LICENCE
 new
 none
-obtaining
-of
 on
 open_source
 option
@@ -180,16 +143,11 @@ option)
 or
 or(at
 others
-person
 public
-released
-software
 style
 terms
 the
-to
 under?
-unencumbered
 unknown
 unrestricted
 uses
@@ -202,3 +160,13 @@ with
 your
 |
 ~
+Permission
+a
+charge
+copy
+granted
+hereby
+obtaining
+of
+person
+to

autospec/license_translations

@@ -16,11 +16,9 @@ Apache License 2.0, Apache-2.0
 Apache License, Version 2.0, Apache-2.0
 Apache, Apache-2.0
 Apache-2, Apache-2.0
-Apache2, Apache-2.0
 Apache2.0, Apache-2.0
 Apachev2, Apache-2.0
 Artistic-1.0+GPL-1.0, Artistic-1.0 GPL-1.0
-Artistic/GPL, Artistic-1.0-Perl GPL-1.0-or-later
 Artistic_2, Artistic-2.0
 BSD(3-clause), BSD-3-Clause
 BSD-2-clause, BSD-2-Clause

autospec/pkg_integrity.py

@@ -42,7 +42,7 @@ KEYID_TRY = ""
 KEYID = ""
 IMPORTED = ""
 EMAIL = ""
-GNUPGCONF = """keyserver keyserver.ubuntu.com"""
+GNUPGCONF = """keyserver keys.gnupg.net"""
 CMD_TIMEOUT = 20
 ENV = os.environ
 INPUT_GETTER_TIMEOUT = 60
@@ -205,9 +205,9 @@ class Verifier(object):
     @staticmethod
     def quit():
         """Stop verification."""
-        util.print_fatal("Verification required for build (verify_required option set)")
+        print('Critical error quitting...')
         print(SEPT)
-        sys.exit(1)
+        exit(1)
 
     @staticmethod
     def calc_sum(filepath, digest_algo):
@@ -264,12 +264,12 @@ def get_signature_file(package_url, package_path):
 def compare_keys(newkey, oldkey):
     """Key comparison to check against key tampering."""
     if newkey != oldkey:
-        util.print_fatal('Public key has changed:\n'
+        util.print_error('Public key has changed:\n'
                          '            old key: {}\n'
                          '            new key: {}\n'
                          'this is a critical security error, quitting...'
                          .format(oldkey, newkey))
-        sys.exit(1)
+        exit(1)
 
 
 # sha256sum Verifier
@@ -521,6 +521,7 @@ class GPGVerifier(Verifier):
 
 def quit_verify():
     """Halt build due to verification being required."""
+    util.print_error("Verification required for build (verify_required option set)")
     Verifier.quit()
 
 
@@ -531,7 +532,6 @@ VERIFIER_TYPES = {
     '.bz2': GPGVerifier,
     '.xz': GPGVerifier,
     '.zip': GPGVerifier,
-    '.zst': GPGVerifier,
 }
 
 

autospec/pypidata.py

@@ -110,7 +110,7 @@ def main():
     pkg_name = sys.argv[1]
     pypi_name = get_pypi_name(pkg_name)
     if not pypi_name:
-        util.print_fatal(f"Couldn't find {pkg_name} in pypi")
+        print(f"Couldn't find {pkg_name} in pypi")
         sys.exit(1)
     pypi_metadata = get_pypi_metadata(pypi_name)
     print(pypi_metadata)

autospec/specfiles.py

@@ -26,8 +26,6 @@ import types
 from collections import OrderedDict
 
 import git
-from jinja2 import Environment
-from jinja2.loaders import DictLoader
 from util import _file_write, open_auto
 
 AVX2_CFLAGS = "-march=x86-64-v3"
@@ -75,42 +73,12 @@ class Specfile(object):
 
     def write_spec(self):
         """Write spec file."""
-        spec_path = f"{os.path.join(self.config.download_path, self.name)}.spec"
-        self.specfile = open_auto(spec_path, "w")
+        self.specfile = open_auto("{}/{}.spec".format(self.config.download_path, self.name), "w")
         self.specfile.write_strip = types.MethodType(_file_write, self.specfile)
 
-        # last chance to sanitize url for template and build types
-        if self.config.urlban:
-            clean_url = re.sub(self.config.urlban, "localhost", self.url)
-            # Duplicate prefixes entry before we change the url
-            self.content.prefixes[clean_url] = self.content.prefixes.get(self.url)
-            self.url = clean_url
-
-        template_path = f"{spec_path}.template"
-
-        if os.path.isfile(template_path):
-            # make templates have a template build pattern
-            self.config.default_pattern = "template"
         # spec file comment header
         self.write_comment_header()
 
-        if os.path.isfile(template_path):
-            with open_auto(template_path) as tfile:
-                template_content = tfile.read()
-            template = Environment(loader=DictLoader({'spec': template_content})).get_template('spec')
-            kw = {
-                'package_name': self.name,
-                'package_version': self.version,
-                'package_url': self.url,
-                'package_release': self.release,
-            }
-            self.specfile.write(template.render(**kw))
-            self.specfile.write_strip('\n')
-            self.specfile.close()
-            # return specfile type built so autospec knows how to
-            # handle build results (template should only builds once)
-            return "template"
-
         if self.config.config_opts.get('keepstatic'):
             self._write("%define keepstatic 1\n")
 
@@ -142,10 +110,6 @@ class Specfile(object):
 
         self.specfile.close()
 
-        # return specfile type built so autospec knows how to
-        # handle build results (generate has multiple builds)
-        return "generate"
-
     def write_comment_header(self):
         """Write comment header to spec file."""
         self._write("#\n")
@@ -168,6 +132,11 @@ class Specfile(object):
 
     def write_nvr(self):
         """Write name, version, and release information."""
+        if self.config.urlban:
+            clean_url = re.sub(self.config.urlban, "localhost", self.url)
+            # Duplicate prefixes entry before we change the url
+            self.content.prefixes[clean_url] = self.content.prefixes.get(self.url)
+            self.url = clean_url
         self._write("Name     : {}\n".format(self.name))
         self._write("Version  : {}\n".format(self.version))
         self._write("Release  : {}\n".format(str(self.release)))
@@ -326,9 +295,6 @@ class Specfile(object):
                 self._write("Group: Default\n")
 
             for dep in deps.get(pkg, []):
-                # honor requires_ban for manual overrides
-                if f"{self.name}-{dep}" in self.requirements.banned_requires.get(pkg, []):
-                    continue
                 if dep in self.packages:
                     self._write("Requires: {}-{} = %{{version}}-%{{release}}\n".format(self.name, dep))
 
@@ -448,8 +414,6 @@ class Specfile(object):
             self._write_strip("## make_prepend end")
         if self.config.make_command:
             make = self.config.make_command
-        elif self.config.config_opts['use_ninja']:
-            make = "ninja"
         else:
             make = "make"
         if build32:
@@ -466,11 +430,7 @@ class Specfile(object):
 
     def write_cmake_line_openmpi(self):
         """Write cmake line (openmpi) to spec file."""
-        if self.config.config_opts['use_ninja']:
-            cmake_type = "Ninja"
-        else:
-            cmake_type = "Unix Makefiles"
-        cmake_string = f"cmake -G '{cmake_type}' -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
+        cmake_string = 'cmake -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n' \
                        '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \
                        '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \
                        '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n'
@@ -486,16 +446,12 @@ class Specfile(object):
             self._write_strip(f"gpg --homedir .gnupg --status-fd 1 --verify {self.config.signature_macro} %{{SOURCE0}} > gpg.status")
             self._write_strip(f"grep -E '^\\[GNUPG:\\] (GOODSIG|EXPKEYSIG) {self.keyid}' gpg.status")
         self.write_prep_prepend()
+        prefix = self.content.prefixes[self.url]
         if self.config.default_pattern == 'R':
             prefix = self.content.tarball_prefix
             self._write_strip("%setup -q -n " + prefix)
         else:
-            prefix = self.content.prefixes[self.url]
-            if not prefix:
-                prefix = os.path.splitext(os.path.basename(self.url))[0]
-                self._write_strip("%setup -q -c -n " + prefix)
-            else:
-                self._write_strip("%setup -q -n " + prefix)
+            self._write_strip("%setup -q -n " + prefix)
             for archive in self.config.sources["archive"]:
                 # Handle various archive types
                 extract_cmd = 'tar xf {}'
@@ -503,11 +459,6 @@ class Specfile(object):
                     extract_cmd = 'unzip -q {}'
                 if archive.endswith('.bz2') and not archive.endswith('.tar.bz2'):
                     extract_cmd = 'bzcat {0} > $(basename "{0}" .bz2)'
-                if archive.endswith('.zst'):
-                    if archive.endswith('.tar.zst'):
-                        extract_cmd = 'tar -I zstd xf {}'
-                    else:
-                        extract_cmd = 'zstd -dqc {0} > $(basename "{0}" .zst)'
                 self._write_strip('cd %{_builddir}')
                 archive_file = os.path.basename(archive)
                 if self.config.archive_details.get(archive + "prefix"):
@@ -540,43 +491,45 @@ class Specfile(object):
                 if not archive_prefix:
                     # Make it up
                     archive_prefix = os.path.splitext(os.path.basename(archive))[0]
-                self._write_strip("cp -r %{{_builddir}}/{0}/. %{{_builddir}}/{1}/{2}"
+                self._write_strip("cp -r %{{_builddir}}/{0}/* %{{_builddir}}/{1}/{2}"
                                   .format(archive_prefix,
                                           self.content.tarball_prefix,
                                           destination))
         self.apply_patches()
 
         # setup cargo.toml vendoring if needed
-        if self.config.cargo_vendors:
+        if self.config.config_opts['cargo_vendor']:
             if self.config.subdir:
                 self._write_strip("pushd " + self.config.subdir)
             self._write_strip("mkdir -p .cargo")
-            self._write_strip(f"echo '\n{self.config.cargo_vendors}' >> .cargo/config.toml")
+            self._write_strip("echo '[source.crates-io]' >> .cargo/config.toml")
+            self._write_strip("""echo 'replace-with = "vendored-sources"' >> .cargo/config.toml""")
+            self._write_strip("echo '[source.vendored-sources]' >> .cargo/config.toml")
+            self._write_strip("""echo 'directory = "vendor"' >> .cargo/config.toml""")
             if self.config.subdir:
                 self._write_strip("popd")
 
-        self.write_copy_prepend()
-
-        if self.config.config_opts['32bit']:
-            self._write_strip("pushd ..")
-            self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
-            self._write_strip("popd")
-        if self.config.config_opts['use_avx2']:
-            self._write_strip("pushd ..")
-            self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
-            self._write_strip("popd")
-        if self.config.config_opts['use_avx512']:
-            self._write_strip("pushd ..")
-            self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
-            self._write_strip("popd")
-        if self.config.config_opts['use_apx']:
-            self._write_strip("pushd ..")
-            self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
-            self._write_strip("popd")
-        if self.config.config_opts['openmpi']:
-            self._write_strip("pushd ..")
-            self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
-            self._write_strip("popd")
+        if self.config.default_pattern != 'cmake':
+            if self.config.config_opts['32bit']:
+                self._write_strip("pushd ..")
+                self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
+                self._write_strip("popd")
+            if self.config.config_opts['use_avx2']:
+                self._write_strip("pushd ..")
+                self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
+                self._write_strip("popd")
+            if self.config.config_opts['use_avx512']:
+                self._write_strip("pushd ..")
+                self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
+                self._write_strip("popd")
+            if self.config.config_opts['use_apx']:
+                self._write_strip("pushd ..")
+                self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
+                self._write_strip("popd")
+            if self.config.config_opts['openmpi']:
+                self._write_strip("pushd ..")
+                self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
+                self._write_strip("popd")
         self._write_strip("\n")
 
     def write_32bit_exports(self):
@@ -638,7 +591,7 @@ class Specfile(object):
             if self.config.config_opts['use_clang']:
                 flags.extend(["-O3"])
             else:
-                flags.extend(["-fno-semantic-interposition", "-falign-functions=32"])
+                flags.extend(["-Ofast", "-fno-semantic-interposition", "-falign-functions=32"])
         if not self.config.config_opts['full-debug-info'] and not self.config.config_opts['use_clang']:
             flags.extend(["-gno-variable-location-views", "-gno-column-info", "-femit-struct-debug-baseonly", "-fdebug-types-section", "-gz=zstd", "-g1"])
         if self.config.default_pattern != 'qmake' or self.config.default_pattern != 'qmake6':
@@ -836,14 +789,6 @@ class Specfile(object):
                 self._write_strip("{}\n".format(line))
             self._write_strip("## prep_prepend end")
 
-    def write_copy_prepend(self):
-        """Write out any custom supplied commands prior to creating source copies for avx, etc builds."""
-        if self.config.copy_prepend:
-            self._write_strip("## copy_prepend content")
-            for line in self.config.copy_prepend:
-                self._write_strip("{}\n".format(line))
-            self._write_strip("## copy_prepend end")
-
     def write_build_prepend(self):
         """Write out any custom supplied commands at the start of the %build section and every build type."""
         if self.config.build_prepend:
@@ -959,11 +904,10 @@ class Specfile(object):
 
         self._write_strip("export GOAMD64=v2")
 
-        if self.config.config_opts['use_ninja'] and self.config.install_macro == '%make_install':
-            self.config.install_macro = '%ninja_install'
+        if self.config.subdir:
+            self._write_strip("pushd " + self.config.subdir)
 
         if self.config.config_opts['32bit']:
-            self._write_strip("pushd ../build32/" + self.config.subdir)
             self._write_strip("pushd clr-build32")
             self._write_strip("{}32 {} {}".format(self.config.install_macro,
                                                   self.config.extra_make_install,
@@ -981,42 +925,30 @@ class Specfile(object):
             self._write_strip("    popd")
             self._write_strip("fi")
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['use_avx2']:
-            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-avx2")
             self._write_strip("%s_v3 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['use_avx512']:
-            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
             self._write_strip("GOAMD64=v4")
             self._write_strip("pushd clr-build-avx512")
             self._write_strip("%s_v4 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['use_apx']:
-            self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-apx")
             self._write_strip("%s_va %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['openmpi']:
-            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-openmpi")
             self.write_install_openmpi()
             self._write_strip("popd")
-            self._write_strip("popd")
-
-        if self.config.subdir:
-            self._write_strip("pushd " + self.config.subdir)
 
         self._write_strip("GOAMD64=v2")
         self._write_strip("pushd clr-build")
@@ -1165,6 +1097,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC="gcc-14"')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1270,6 +1203,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC="gcc-14"')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1330,6 +1264,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1348,8 +1283,6 @@ class Specfile(object):
         self.write_lang_c(export_epoch=True)
         self.write_variables()
         self.write_profile_payload("autogen")
-        if self.config.subdir:
-            self._write_strip("pushd " + self.config.subdir)
         self._write_strip("export GOAMD64=v2")
         self._write_strip("{0}%autogen {1} {2} {3}"
                           .format(self.get_profile_use_flags(),
@@ -1358,8 +1291,6 @@ class Specfile(object):
                                   self.config.extra_configure64))
         self.write_make_line()
         self._write_strip("\n")
-        if self.config.subdir:
-            self._write_strip("popd")
         if self.config.config_opts['32bit']:
             self._write_strip("pushd ../build32/" + self.config.subdir)
             self.write_build_prepend()
@@ -1411,6 +1342,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1437,9 +1369,6 @@ class Specfile(object):
         for module in self.config.pypi_overrides:
             self._write_strip(f"pypi-dep-fix.py . {module}")
         self._write_strip("python3 -m build --wheel --skip-dependency-check --no-isolation " + self.config.extra_configure)
-        self._write_strip("\n")
-        if self.config.subdir:
-            self._write_strip("popd")
 
         if self.config.config_opts['use_avx2']:
             self._write_strip("pushd ../buildavx2/" + self.config.subdir)
@@ -1458,6 +1387,7 @@ class Specfile(object):
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1470,6 +1400,8 @@ class Specfile(object):
             self._write_strip("popd")
 
         self._write_strip("\n")
+        if self.config.subdir:
+            self._write_strip("popd")
         self.write_build_append()
         self.write_check()
         self._write_strip("%install")
@@ -1503,6 +1435,7 @@ class Specfile(object):
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1545,6 +1478,7 @@ class Specfile(object):
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1589,6 +1523,7 @@ class Specfile(object):
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1704,22 +1639,14 @@ class Specfile(object):
         self._write_strip("pushd clr-build")
         self.write_variables()
         self._write_strip("export GOAMD64=v2")
-        if self.config.config_opts['use_ninja']:
-            cmake_type = "-G Ninja"
-        else:
-            cmake_type = "-G 'Unix Makefiles'"
-        self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+        self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
 
         self.write_profile_payload("cmake")
 
         self.write_make_line()
         self._write_strip("popd")
 
-        if self.config.subdir:
-            self._write_strip("popd")
-
         if self.config.config_opts['use_avx2']:
-            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-avx2")
             self._write_strip("pushd clr-build-avx2")
             self.write_build_prepend()
@@ -1729,13 +1656,11 @@ class Specfile(object):
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {AVX2_CFLAGS} "')
-            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['use_avx512']:
-            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-avx512")
             self._write_strip("pushd clr-build-avx512")
             self.write_build_prepend()
@@ -1745,29 +1670,26 @@ class Specfile(object):
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX512_CFLAGS} {AVX512_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {AVX512_CFLAGS} {AVX512_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {AVX512_CFLAGS} "')
-            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
-            self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-apx")
             self._write_strip("pushd clr-build-apx")
             self.write_build_prepend()
             self.write_variables()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {APX_CFLAGS} "')
-            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['32bit']:
-            self._write_strip("pushd ../build32/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build32")
             self._write_strip("pushd clr-build32")
             self.write_build_prepend()
@@ -1776,14 +1698,12 @@ class Specfile(object):
             self._write_strip("%cmake -DLIB_INSTALL_DIR:PATH=/usr/lib32 "
                               "-DCMAKE_INSTALL_LIBDIR=/usr/lib32 "
                               "-DLIB_SUFFIX=32 "
-                              f"{self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+                              "{} {} ".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("unset PKG_CONFIG_PATH")
             self._write_strip("popd")
-            self._write_strip("popd")
 
         if self.config.config_opts['openmpi']:
-            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-openmpi")
             self._write_strip("pushd clr-build-openmpi")
             self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh")
@@ -1799,6 +1719,8 @@ class Specfile(object):
             self.write_make_line()
             self._write_strip("module unload openmpi")
             self._write_strip("popd")
+
+        if self.config.subdir:
             self._write_strip("popd")
 
         self._write_strip("\n")
@@ -1833,9 +1755,6 @@ class Specfile(object):
         self._write_strip('export QMAKE_CFLAGS_RELEASE=')
         self._write_strip('export QMAKE_CXXFLAGS_RELEASE=')
 
-        # Add the qt6base tools to the path
-        self._write_strip('export PATH=/usr/lib64/qt6/bin:$PATH')
-
         if self.config.make_command:
             qmake = self.config.make_command
         else:
@@ -1967,6 +1886,7 @@ class Specfile(object):
                 self._write_strip('ninja -v -C builddiravx512')
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             if self.config.config_opts['pgo'] and self.config.profile_payload != "":
                 self._write_strip(f'CFLAGS="$CFLAGS_GENERATE {APX_CFLAGS} {APX_LFLAGS} "'
                                   f' CXXFLAGS="$CXXFLAGS_GENERATE {AVX2_CFLAGS} {AVX2_LFLAGS} "'
@@ -2037,6 +1957,7 @@ class Specfile(object):
             self._write_strip('DESTDIR=%{buildroot}-v4 ninja -C builddiravx512 install')
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip('DESTDIR=%{buildroot}-va ninja -C builddirapx install')
 
         self._write_strip("GOAMD64=v2")
@@ -2070,11 +1991,14 @@ class Specfile(object):
         self.write_license_files()
         if self.config.subdir:
             self._write_strip("pushd " + self.config.subdir)
-        self._write_strip("cargo install --path .")
-        self._write_strip("mkdir -p %{buildroot}/usr/bin")
-        self._write_strip('pushd "${HOME}/.cargo/bin/"')
-        self._write_strip("mv * %{buildroot}/usr/bin/")
-        self._write_strip("popd")
+        if self.config.install_macro:
+            self._write_strip(self.config.install_macro)
+        else:
+            self._write_strip("cargo install --path .")
+            self._write_strip("mkdir -p %{buildroot}/usr/bin")
+            self._write_strip('pushd "${HOME}/.cargo/bin/"')
+            self._write_strip("mv * %{buildroot}/usr/bin/")
+            self._write_strip("popd")
         if self.config.subdir:
             self._write_strip("popd")
         self.write_install_append()

autospec/tarball.py

@@ -26,8 +26,7 @@ import tarfile
 import zipfile
 
 import download
-import zstandard as zstd
-from util import do_regex, get_sha1sum, print_fatal, write_out
+from util import call, do_regex, get_sha1sum, print_fatal, write_out
 
 
 class Source():
@@ -54,8 +53,6 @@ class Source():
             self.type = 'zip'
         elif self.url.lower().endswith(('.bz2')) and not self.url.lower().endswith(('.tar.bz2')):
             self.type = 'bz2'
-        elif self.url.lower().endswith('.zst'):
-            self.type = 'zst'
         else:
             self.type = 'tar'
 
@@ -84,16 +81,6 @@ class Source():
             print_fatal("Not a valid tar file.")
             sys.exit(1)
 
-    def set_zst_prefix(self):
-        """Determine prefix folder name of tar.zst file."""
-        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
-            lines = content.getnames()
-            if len(lines) == 0:
-                print_fatal("Zstd compressed tar file doesn't appear to have any content")
-                sys.exit(1)
-            elif len(lines) > 1:
-                self.prefix = os.path.commonpath(lines)
-
     def set_bz2_prefix(self):
         """No prefix for plain bz2 archives."""
 
@@ -120,15 +107,12 @@ class Source():
             extraction_path = base_path
 
         extract_method = getattr(self, 'extract_{}'.format(self.type))
-        try:
-            extract_method(extraction_path)
-        except tarfile.AbsoluteLinkError:
-            pass
+        extract_method(extraction_path)
 
     def extract_tar(self, extraction_path):
         """Extract tar in path."""
         with tarfile.open(self.path) as content:
-            content.extractall(path=extraction_path, filter='data')
+            content.extractall(path=extraction_path)
 
     def extract_bz2(self, extraction_path):
         """Extract plain bz2 file in path."""
@@ -144,11 +128,6 @@ class Source():
         with zipfile.ZipFile(self.path, 'r') as content:
             content.extractall(path=extraction_path)
 
-    def extract_zst(self, extraction_path):
-        """Extract zst in path."""
-        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
-            content.extractall(path=extraction_path, filter='data')
-
 
 def convert_version(ver_str, name):
     """Remove disallowed characters from the version."""
@@ -156,7 +135,7 @@ def convert_version(ver_str, name):
     # them out with expensive regular expressions
     banned_subs = ["x86.64", "source", "src", "all", "bin", "release", "rh",
                    "ga", ".ce", "lcms", "onig", "linux", "gc", "sdk", "orig",
-                   "jurko", "%2f", "%2F", "%20", "x265", "autotools"]
+                   "jurko", "%2f", "%2F", "%20", "x265"]
 
     # package names may be modified in the version string by adding "lib" for
     # example. Remove these from the name before trying to remove the name from
@@ -172,7 +151,7 @@ def convert_version(ver_str, name):
         ver_str = ver_str.replace(name.replace(mod, ""), "")
 
     # replace illegal characters
-    ver_str = ver_str.strip().replace('-', '.').replace('_', '.').replace('+', '.')
+    ver_str = ver_str.strip().replace('-', '.').replace('_', '.')
 
     # remove banned substrings
     for sub in banned_subs:
@@ -205,6 +184,7 @@ class Content():
         self.prefixes = dict()
         self.config = config
         self.base_path = base_path
+        self.autogenerated_tarball = None
 
     def write_upstream(self, sha, tarfile, mode="w"):
         """Write the upstream hash to the upstream file."""
@@ -234,6 +214,14 @@ class Content():
         main_src = Source(url, '', src_path, self.config.default_pattern)
         return main_src
 
+    def process_autogenerated_source(self, url):
+        """Download any autogenerated source tarball for comparison."""
+        autogenerated_src = None
+        if url:
+            src_path = self.check_or_get_file(url, os.path.basename(url), mode="")
+            autogenerated_src = Source(url, '../autogenerated-tmp', src_path, self.config.default_pattern)
+        return autogenerated_src
+
     def print_header(self):
         """Print header for autospec run."""
         print("\n")
@@ -298,8 +286,7 @@ class Content():
         if "github.com" in self.url:
             # define regex accepted for valid packages, important for specific
             # patterns to come before general ones
-            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/.*/(.*).tar",
-                               r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
+            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
@@ -317,6 +304,9 @@ class Content():
                     name = re.sub(r"release-", '', name)
                     name = re.sub(r"\d*$", '', name)
                 self.rawname = name
+                # Identify the auto-generated tarball URL for comparison
+                if "/releases/download/" in self.url:
+                    self.autogenerated_tarball = "https://github.com/" + match.group(1).strip() + "/" + self.repo + "/archive/refs/tags/" + match.group(3).strip() + ".tar.gz"
                 version = match.group(3).replace(name, '')
                 if "/archive/" not in self.url:
                     version = re.sub(r"^[-_.a-zA-Z]+", "", version)
@@ -449,3 +439,15 @@ class Content():
         archives_src = self.process_archives()
         # Extract all sources
         self.extract_sources(main_src, archives_src)
+        # Download and process any auto-generated source-tree archive for comparison
+        autogenerated_src = self.process_autogenerated_source(self.autogenerated_tarball)
+        # Extract autogenerated source for comparison
+        if autogenerated_src:
+            autogenerated_src.extract(os.path.join(self.base_path, 'autogenerated-tmp'))
+            # Move the autogenerated source to a non-version-named directory for consistent diffs
+            call(f"mv autogenerated-tmp/{autogenerated_src.prefix} autogenerated", check=True, cwd=self.base_path)
+            call("diff -u -r --unidirectional-new-file ../autogenerated ./",
+                 logfile="archive.diff.in", check=False, cwd=os.path.join(self.base_path, main_src.prefix))
+            call("grep -A14 -E '^(diff|Only in)' archive.diff.in",
+                 logfile="archive.diff", check=False, cwd=os.getcwd())
+            call("rm archive.diff.in", check=False, cwd=os.getcwd())

autospec/util.py

@@ -27,101 +27,6 @@ import sys
 dictionary_filename = os.path.dirname(__file__) + "/translate.dic"
 dictionary = [line.strip() for line in open(dictionary_filename, 'r')]
 os_paths = None
-ERROR_FILE = 'pumpAutospec'
-ERROR_ENV = 'AUTOSPEC_UPDATE'
-
-
-def _log_error(error):
-    write_out(ERROR_FILE, f"{error}\n", mode='a')
-
-
-def _commit_result():
-    if not os.path.isfile(ERROR_FILE):
-        return
-    call(f"git add {ERROR_FILE}", check=False, stderr=subprocess.DEVNULL)
-    call(f"git commit {ERROR_FILE} -m 'Notes update'", check=False, stderr=subprocess.DEVNULL)
-    call("git push", check=False, stderr=subprocess.DEVNULL)
-
-
-def _process_line(line, prev_line, current_patch, reported_patches, error):
-    if m := re.match('^Patch #[0-9]+ .(?P<patch>.*).:', line):
-        current_patch[0] = m.group('patch')
-
-    if m := re.match('Hunk #[0-9]+ FAILED at [0-9]+', line):
-        if current_patch[0] not in reported_patches:
-            _log_error("Patch " + current_patch[0] + " does not apply")
-            reported_patches[current_patch[0]] = True
-            return True
-
-    if m := re.match(".*can't find file to patch at input line ", line):
-        if current_patch[0] not in reported_patches:
-            _log_error("Patch " + current_patch[0] + " does not apply")
-            reported_patches[current_patch[0]] = True
-            return True
-
-    if m := re.match('.*meson.build:[0-9]+:[0-9]+: ERROR: Unknown options: "(?P<option>.*)"', line):
-        _log_error("Unknown meson option: " + m.group('option'))
-        return True
-
-    if m := re.match('Error: package ‘(?P<module>.*)’ .* was found, but', line):
-        _log_error("R package " + m.group('module') + " not found")
-        return True
-
-    if m := re.match('.*CMake Error at .*/CMake', prev_line):
-        if m := re.match('(?P<module>.*) not found', line):
-            _log_error("CMake module " + m.group('module') + "not found")
-            return True
-
-    if m := re.match(r'go: download.*connect: connection refused', line):
-        _log_error("Go online update")
-        return True
-
-    if 'Updating crates.io index' in line:
-        _log_error("Rust crates.io online update")
-        return True
-
-    if "error: '__builtin_ctzs' needs isa option -mbmi" in line:
-        _log_error(" error: '__builtin_ctzs' needs isa option -mbmi")
-        return True
-
-    if "error:" in line and 'Bad exit status from' not in line:
-        m = re.match('.*error:(?P<error>.*)', line)
-        if m and not error:
-            _log_error("Compiler: " + m.group('error'))
-            return True
-
-    if m := re.match(r'Could NOT find (?P<package>.*) .missing', line):
-        _log_error("CMake module " + m.group('package') + " not found")
-        return True
-    if m := re.match(r'Could not find a package configuration file provided by (?P<package>.*) with', line):
-        _log_error("CMake module " + m.group('package') + " not found")
-        return True
-    # Unable to find program 'gperf'
-    if m := re.match(r"Failed to find program ‘(?P<module>.*)’", line):
-        _log_error("Failed to find " + m.group('module'))
-        return True
-    if m := re.match(r"Failed to find ‘(?P<module>.*)’", line):
-        _log_error("Failed to find " + m.group('module'))
-        return True
-
-    return False
-
-
-def _process_build_log(filename):
-    with open_auto(filename, "r") as lfile:
-        lines = lfile.readlines()
-
-    prev_line = ''
-    current_patch = ['']
-    reported_patches = {}
-    error = False
-    for line in lines:
-        if _process_line(line, prev_line, current_patch, reported_patches, error):
-            error = True
-        prev_line = line
-
-    if error:
-        _commit_result()
 
 
 def call(command, logfile=None, check=True, **kwargs):
@@ -211,22 +116,9 @@ def print_error(message):
     _print_message(message, 'ERROR', 'red')
 
 
-def print_build_failed():
-    """Print final fatal error, color coded for TTYs."""
-    _print_message('Build failed, aborting', 'FATAL', 'red')
-    try:
-        if os.environ.get(ERROR_ENV):
-            _process_build_log('results/build.log')
-    except Exception:
-        pass
-
-
 def print_fatal(message):
     """Print fatal error, color coded for TTYs."""
     _print_message(message, 'FATAL', 'red')
-    if os.environ.get(ERROR_ENV):
-        write_out(ERROR_FILE, f"{message}\n", mode='a')
-        _commit_result()
 
 
 def print_warning(message):
@@ -258,8 +150,9 @@ def binary_in_path(binary):
 
 def write_out(filename, content, mode="w"):
     """File.write convenience wrapper."""
-    with open_auto(filename, mode) as require_f:
-        require_f.write(content)
+    if mode:
+        with open_auto(filename, mode) as require_f:
+            require_f.write(content)
 
 
 def open_auto(*args, **kwargs):
@@ -274,31 +167,3 @@ def open_auto(*args, **kwargs):
     assert 'encoding' not in kwargs
     assert 'errors' not in kwargs
     return open(*args, encoding="utf-8", errors="surrogateescape", **kwargs)
-
-
-def globlike_match(filename, match_name):
-    """Compare the filename to the match_name in a way that simulates the shell glob '*'."""
-    fsplit = filename.split('/')
-    if len(fsplit) != len(match_name):
-        return False
-    match = True
-    for fpart, mpart in zip(fsplit, match_name):
-        if fpart != mpart:
-            if '*' not in mpart:
-                match = False
-                break
-            if len(mpart) > len(fpart) + 1:
-                match = False
-                break
-            mpl, mpr = mpart.split('*')
-            try:
-                if fpart.index(mpl) != 0:
-                    match = False
-                    break
-                if fpart.rindex(mpr) != len(fpart) - len(mpr):
-                    match = False
-                    break
-            except ValueError:
-                match = False
-                break
-    return match

tests/packageurls

@@ -1612,5 +1612,3 @@ http://sourceforge.net/projects/zsh/files/zsh/5.4.2/zsh-5.4.2.tar.gz,zsh,5.4.2
 https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3.11-pigeonhole-0.5.11.tar.gz,pigeonhole,0.5.11
 https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3-pigeonhole-0.5.20.tar.gz,pigeonhole,0.5.20
 https://www.ezix.org/software/files/lshw-B.02.19.2.tar.gz,lshw,02.19.2
-https://github.com/VectorCamp/vectorscan/archive/refs/tags/vectorscan/5.4.11.tar.gz,vectorscan,5.4.11
-https://lib.openmpt.org/files/libopenmpt/src/libopenmpt-0.7.13+release.autotools.tar.gz,libopenmpt,0.7.13

tests/test_build.py

@@ -190,36 +190,25 @@ class TestBuildpattern(unittest.TestCase):
         def mock_util_call(cmd):
             del cmd
 
-        def mock_wrap_fatal():
-            output = ['']
-            def mock_print_fatal(msg):
-                output[0] = msg
-            return output, mock_print_fatal
-
-        result, mock_print_fatal = mock_wrap_fatal()
-
-        exit_backup = build.sys.exit
         call_backup = build.util.call
-        print_backup = build.util.print_fatal
-        build.sys.exit = MagicMock()
         build.util.call = mock_util_call
-        build.util.print_fatal = mock_print_fatal
 
         open_name = 'build.util.open_auto'
         content = "line1\nDEBUG util.py:399:  No matching package to install: 'foobar'\nDEBUG util.py:399:  No matching package to install: 'foobarbaz'\nline 4"
         m_open = mock_open(read_data=content)
         pkg = build.Build()
+        pkg.must_restart = 1
+        pkg.file_restart = 1
 
+        result = True
         with patch(open_name, m_open, create=True):
-            pkg.parse_buildroot_log('testname', 1)
+            result = pkg.parse_buildroot_log('testname', 1)
 
-        build.util.print_fatal = print_backup
         build.util.call = call_backup
-        mock_exit = build.sys.exit
-        build.sys.exit = exit_backup
 
-        self.assertEqual(result[0], 'Cannot resolve dependency name: foobar\nCannot resolve dependency name: foobarbaz')
-        mock_exit.assert_called_once()
+        self.assertFalse(result)
+        self.assertEqual(pkg.must_restart, 0)
+        self.assertEqual(pkg.file_restart, 0)
 
     def test_parse_buildroot_log_pass(self):
         """
@@ -238,10 +227,11 @@ class TestBuildpattern(unittest.TestCase):
 
         result = True
         with patch(open_name, m_open, create=True):
-            pkg.parse_buildroot_log('testname', 1)
+            result = pkg.parse_buildroot_log('testname', 1)
 
         build.util.call = call_backup
 
+        self.assertTrue(result)
         self.assertEqual(pkg.must_restart, 0)
 
     def test_parse_buildroot_log_noop(self):

tests/test_buildreq.py

@@ -402,7 +402,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'setup.py'), 'w').close()
@@ -419,7 +418,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'CMakeLists.txt'), 'w').close()
@@ -436,7 +434,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'SConstruct'), 'w').close()
@@ -453,7 +450,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'meson.build'), 'w').close()
@@ -484,7 +480,6 @@ class TestBuildreq(unittest.TestCase):
         buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
-            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
             self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
@@ -517,7 +512,6 @@ class TestBuildreq(unittest.TestCase):
         m_open = mock_open(read_data=content)
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
-            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
             open(os.path.join(tmpd, 'pypi.json'), 'w').close()
@@ -543,7 +537,6 @@ class TestBuildreq(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
-            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
             open(os.path.join(tmpd, 'subdir', 'requires.txt'), 'w').close()
@@ -553,21 +546,6 @@ class TestBuildreq(unittest.TestCase):
         self.reqs.add_setup_py_requires.assert_called_once()
         self.reqs.grab_python_requirements.assert_called_once()
 
-    def test_scan_for_configure_ninja(self):
-        """
-        Test scan_for_configure when ninja is enabled.
-        """
-        conf = config.Config("")
-        conf.config_opts['use_ninja'] = True
-        with tempfile.TemporaryDirectory() as tmpd:
-            os.mkdir(os.path.join(tmpd, 'subdir'))
-            open(os.path.join(tmpd, 'setup.py'), 'w').close()
-
-            self.reqs.scan_for_configure(tmpd, "", conf)
-
-        self.assertEqual(self.reqs.buildreqs,
-                         set(['buildreq-distutils3', 'ninja']))
-
     def test_parse_cmake_pkg_check_modules(self):
         """
         Test parse_cmake to ensure accurate detection of versioned and

tests/test_check.py

@@ -152,6 +152,26 @@ class TestTest(unittest.TestCase):
         check.os.listdir = listdir_backup
         self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test')
 
+    def test_scan_for_tests_setup(self):
+        """
+        Test scan_for_tests with setup.py suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['setup.py'])
+        content = 'test_suite'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "distutils3"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") '
+                         'python setup.py test')
+
     def test_scan_for_tests_cmake(self):
         """
         Test scan_for_tests with cmake suite

tests/test_config.py

@@ -6,7 +6,6 @@ import config
 # Structure: (url, build_pattern)
 BUILD_PAT_URL = [
     ("https://cran.r-project.org/src/contrib/raster_3.0-12.tar.gz", "R"),
-    ("https://ftp.osuosl.org/pub/cran/src/contrib/hexbin_1.28.5.tar.gz", "R"),
     ("http://pypi.debian.net/argparse/argparse-1.4.0.tar.gz", "distutils3"),
     ("https://pypi.python.org/packages/source/T/Tempita/Tempita-0.5.2.tar.gz", "distutils3"),
     ("https://cpan.metacpan.org/authors/id/T/TO/TODDR/IO-Tty-1.14.tar.gz", "cpan"),

tests/test_license.py

@@ -71,7 +71,7 @@ class TestLicense(unittest.TestCase):
         conf = config.Config("")
         conf.setup_patterns()
         # remove the hash from license_hashes
-        del(conf.license_hashes[license.util.get_sha1sum('tests/COPYING_TEST')])
+        del(conf.license_hashes[license.get_sha1sum('tests/COPYING_TEST')])
         conf.license_show = "license.show.url"
         license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
 
@@ -233,62 +233,6 @@ class TestLicense(unittest.TestCase):
         self.assertIn("Cannot find any license", out.getvalue())
         self.assertEqual(license.licenses, [])
 
-    def test_scan_for_licenses_skip(self):
-        """
-        Test scan_for_licenses in temporary directory with licenses to skip
-        """
-        conf = config.Config("")
-        conf.setup_patterns()
-        conf.license_skips = [['COPYING']]
-        with open('tests/COPYING_TEST', 'rb') as copyingf:
-            content = copyingf.read()
-
-        with tempfile.TemporaryDirectory() as tmpd:
-            # create the copying file
-            with open(os.path.join(tmpd, 'COPYING'), 'w') as newcopyingf:
-                newcopyingf.write(content.decode('utf-8'))
-            # create some cruft for testing
-            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
-                with open(os.path.join(tmpd, testf), 'w') as newtestf:
-                    newtestf.write('test content')
-            # let's check that the proper thing is being printed as well
-            out = StringIO()
-            with redirect_stdout(out):
-                with self.assertRaises(SystemExit) as thread:
-                    license.scan_for_licenses(tmpd, conf, '')
-
-        self.assertEqual(thread.exception.code, 1)
-        self.assertIn("Cannot find any license", out.getvalue())
-        self.assertEqual(license.licenses, [])
-
-    def test_scan_for_licenses_skip_prefix_slash(self):
-        """
-        Test scan_for_licenses in temporary directory with licenses to skip
-        """
-        conf = config.Config("")
-        conf.setup_patterns()
-        conf.license_skips = [['', 'COPYING']]
-        with open('tests/COPYING_TEST', 'rb') as copyingf:
-            content = copyingf.read()
-
-        with tempfile.TemporaryDirectory() as tmpd:
-            # create the copying file
-            with open(os.path.join(tmpd, 'COPYING'), 'w') as newcopyingf:
-                newcopyingf.write(content.decode('utf-8'))
-            # create some cruft for testing
-            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
-                with open(os.path.join(tmpd, testf), 'w') as newtestf:
-                    newtestf.write('test content')
-            # let's check that the proper thing is being printed as well
-            out = StringIO()
-            with redirect_stdout(out):
-                with self.assertRaises(SystemExit) as thread:
-                    license.scan_for_licenses(tmpd, conf, '')
-
-        self.assertEqual(thread.exception.code, 1)
-        self.assertIn("Cannot find any license", out.getvalue())
-        self.assertEqual(license.licenses, [])
-
     def test_load_specfile(self):
         """
         Test load_specfile with populated license list. This method is not

tests/test_specfile.py

@@ -47,7 +47,7 @@ class TestSpecfileWrite(unittest.TestCase):
         self.WRITES = self.WRITES[:4] + self.WRITES[6:]
         self.assertEqual(expect, self.WRITES)
 
-    def test_write_nvr(self):
+    def test_write_nvr_no_urlban(self):
         """
         test Specfile.write_nvr with no urlban set
         """
@@ -59,6 +59,19 @@ class TestSpecfileWrite(unittest.TestCase):
                   "Source0  : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n"]
         self.assertEqual(expect, self.WRITES)
 
+    def test_write_nvr_urlban(self):
+        """
+        test Specfile.write_nvr with urlban set
+        """
+        self.specfile.config.urlban = "www.testpkg.com"
+        self.specfile.write_nvr()
+        expect = ["Name     : pkg\n",
+                  "Version  : 1.0\n",
+                  "Release  : 2\n",
+                  "URL      : http://localhost/testpkg/pkg-1.0.tar.gz\n",
+                  "Source0  : http://localhost/testpkg/pkg-1.0.tar.gz\n"]
+        self.assertEqual(expect, self.WRITES)
+
     def test_write_sources(self):
         """
         test write_sources with all Specfile.sources set.

tests/test_util.py

@@ -2,8 +2,7 @@ import subprocess
 import os
 import tempfile
 import unittest
-from unittest.mock import MagicMock, mock_open, patch
-
+import unittest.mock
 import util
 
 
@@ -65,71 +64,5 @@ class TestUtil(unittest.TestCase):
             self.assertTrue(util.binary_in_path('testbin'))
             self.assertEqual(util.os_paths, [tmpd])
 
-    def test__process_build_log_bad_patch(self):
-        """
-        Test _process_build_log with a bad patch
-        """
-        def isfile_mock(_):
-            return True
-        isfile_backup = util.os.path.isfile
-        util.os.path.isfile = isfile_mock
-        call_backup = util.call
-        util.call = MagicMock()
-        open_name = 'util.open_auto'
-        content = "Patch #1 (bad.patch):\nHunk #1 FAILED at 1."
-        m_open = mock_open(read_data=content)
-        with patch(open_name, m_open, create=True):
-            util._process_build_log('filename')
-
-        util.os.path.isfile = isfile_backup
-        mock_call = util.call
-        util.call = call_backup
-        self.assertTrue(len(mock_call.mock_calls) == 3)
-
-    def test_globlike_match(self):
-        """
-        Test globlike_match
-        """
-        match_name = ['a', 'b', 'c']
-        file_path = 'a/b'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', 'c']
-        file_path = 'a/b'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', 'bb*']
-        file_path = 'a/b'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', 'b*']
-        file_path = 'a/ab'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', '*a']
-        file_path = 'a/ab'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', 'c*']
-        file_path = 'a/b'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', '*c']
-        file_path = 'a/b'
-        self.assertFalse(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', 'b*']
-        file_path = 'a/b'
-        self.assertTrue(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', '*b']
-        file_path = 'a/b'
-        self.assertTrue(util.globlike_match(file_path, match_name))
-
-        match_name = ['a', 'b']
-        file_path = 'a/b'
-        self.assertTrue(util.globlike_match(file_path, match_name))
-
-
 if __name__ == '__main__':
     unittest.main(buffer=True)