Limit diff to ~10 lines

Also include content of added files.

autospec/buildreq.py

@@ -751,10 +751,6 @@ class Requirements(object):
         configure_ac_files = []
         qmake_profiles = []
         cmake_files = []
-
-        if config.config_opts['use_ninja']:
-            self.add_buildreq('ninja')
-
         for dirpath, _, files in os.walk(dirn):
             default_score = 2 if dirpath == dirn else 1
 

autospec/config.py

@@ -187,7 +187,6 @@ class Config(object):
             "server": "Package is only used by servers",
             "no_glob": "Do not use the replacement pattern for file matching",
             "allow_exe": "Allow Windows executables (*.exe, *.dll) to be packaged",
-            "use_ninja": "Use ninja build files",
         }
         # simple_pattern_pkgconfig patterns
         # contains patterns for parsing build.log for missing dependencies

autospec/git.py

@@ -113,6 +113,7 @@ def commit_to_git(config, name, success):
     call("git add whatrequires", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add description", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add attrs", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add archive.diff", check=False, stderr=subprocess.DEVNULL, cwd=path)
 
     # remove deprecated config files
     call("git rm make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)

autospec/specfiles.py

@@ -414,8 +414,6 @@ class Specfile(object):
             self._write_strip("## make_prepend end")
         if self.config.make_command:
             make = self.config.make_command
-        elif self.config.config_opts['use_ninja']:
-            make = "ninja"
         else:
             make = "make"
         if build32:
@@ -432,11 +430,7 @@ class Specfile(object):
 
     def write_cmake_line_openmpi(self):
         """Write cmake line (openmpi) to spec file."""
-        if self.config.config_opts['use_ninja']:
-            cmake_type = "Ninja"
-        else:
-            cmake_type = "Unix Makefiles"
-        cmake_string = f"cmake -G {cmake_type} -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
+        cmake_string = 'cmake -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n' \
                        '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \
                        '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \
                        '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n'
@@ -597,7 +591,7 @@ class Specfile(object):
             if self.config.config_opts['use_clang']:
                 flags.extend(["-O3"])
             else:
-                flags.extend(["-fno-semantic-interposition", "-falign-functions=32"])
+                flags.extend(["-Ofast", "-fno-semantic-interposition", "-falign-functions=32"])
         if not self.config.config_opts['full-debug-info'] and not self.config.config_opts['use_clang']:
             flags.extend(["-gno-variable-location-views", "-gno-column-info", "-femit-struct-debug-baseonly", "-fdebug-types-section", "-gz=zstd", "-g1"])
         if self.config.default_pattern != 'qmake' or self.config.default_pattern != 'qmake6':
@@ -913,9 +907,6 @@ class Specfile(object):
         if self.config.subdir:
             self._write_strip("pushd " + self.config.subdir)
 
-        if self.config.config_opts['use_ninja'] and self.config.install_macro == '%make_install':
-            self.config.install_macro = '%ninja_install'
-
         if self.config.config_opts['32bit']:
             self._write_strip("pushd clr-build32")
             self._write_strip("{}32 {} {}".format(self.config.install_macro,
@@ -1106,6 +1097,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC="gcc-14"')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1211,6 +1203,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC="gcc-14"')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1271,6 +1264,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1348,6 +1342,7 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1392,6 +1387,7 @@ class Specfile(object):
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1439,6 +1435,7 @@ class Specfile(object):
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1481,6 +1478,7 @@ class Specfile(object):
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1525,6 +1523,7 @@ class Specfile(object):
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1640,11 +1639,7 @@ class Specfile(object):
         self._write_strip("pushd clr-build")
         self.write_variables()
         self._write_strip("export GOAMD64=v2")
-        if self.config.config_opts['use_ninja']:
-            cmake_type = "-G Ninja"
-        else:
-            cmake_type = "-G 'Unix Makefiles'"
-        self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+        self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
 
         self.write_profile_payload("cmake")
 
@@ -1661,7 +1656,7 @@ class Specfile(object):
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {AVX2_CFLAGS} "')
-            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("popd")
 
@@ -1675,7 +1670,7 @@ class Specfile(object):
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX512_CFLAGS} {AVX512_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {AVX512_CFLAGS} {AVX512_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {AVX512_CFLAGS} "')
-            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("popd")
 
@@ -1685,11 +1680,12 @@ class Specfile(object):
             self.write_build_prepend()
             self.write_variables()
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {APX_CFLAGS} "')
-            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("popd")
 
@@ -1702,7 +1698,7 @@ class Specfile(object):
             self._write_strip("%cmake -DLIB_INSTALL_DIR:PATH=/usr/lib32 "
                               "-DCMAKE_INSTALL_LIBDIR=/usr/lib32 "
                               "-DLIB_SUFFIX=32 "
-                              f"{self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
+                              "{} {} ".format(self.config.cmake_srcdir, self.extra_cmake))
             self.write_make_line()
             self._write_strip("unset PKG_CONFIG_PATH")
             self._write_strip("popd")
@@ -1890,6 +1886,7 @@ class Specfile(object):
                 self._write_strip('ninja -v -C builddiravx512')
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             if self.config.config_opts['pgo'] and self.config.profile_payload != "":
                 self._write_strip(f'CFLAGS="$CFLAGS_GENERATE {APX_CFLAGS} {APX_LFLAGS} "'
                                   f' CXXFLAGS="$CXXFLAGS_GENERATE {AVX2_CFLAGS} {AVX2_LFLAGS} "'
@@ -1960,6 +1957,7 @@ class Specfile(object):
             self._write_strip('DESTDIR=%{buildroot}-v4 ninja -C builddiravx512 install')
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("GOAMD64=v3")
+            self._write_strip('CC=gcc-14')
             self._write_strip('DESTDIR=%{buildroot}-va ninja -C builddirapx install')
 
         self._write_strip("GOAMD64=v2")

autospec/tarball.py

@@ -26,7 +26,7 @@ import tarfile
 import zipfile
 
 import download
-from util import do_regex, get_sha1sum, print_fatal, write_out
+from util import call, do_regex, get_sha1sum, print_fatal, write_out
 
 
 class Source():
@@ -184,6 +184,7 @@ class Content():
         self.prefixes = dict()
         self.config = config
         self.base_path = base_path
+        self.autogenerated_tarball = None
 
     def write_upstream(self, sha, tarfile, mode="w"):
         """Write the upstream hash to the upstream file."""
@@ -213,6 +214,14 @@ class Content():
         main_src = Source(url, '', src_path, self.config.default_pattern)
         return main_src
 
+    def process_autogenerated_source(self, url):
+        """Download any autogenerated source tarball for comparison."""
+        autogenerated_src = None
+        if url:
+            src_path = self.check_or_get_file(url, os.path.basename(url), mode="")
+            autogenerated_src = Source(url, '../autogenerated-tmp', src_path, self.config.default_pattern)
+        return autogenerated_src
+
     def print_header(self):
         """Print header for autospec run."""
         print("\n")
@@ -295,6 +304,9 @@ class Content():
                     name = re.sub(r"release-", '', name)
                     name = re.sub(r"\d*$", '', name)
                 self.rawname = name
+                # Identify the auto-generated tarball URL for comparison
+                if "/releases/download/" in self.url:
+                    self.autogenerated_tarball = "https://github.com/" + match.group(1).strip() + "/" + self.repo + "/archive/refs/tags/" + match.group(3).strip() + ".tar.gz"
                 version = match.group(3).replace(name, '')
                 if "/archive/" not in self.url:
                     version = re.sub(r"^[-_.a-zA-Z]+", "", version)
@@ -427,3 +439,15 @@ class Content():
         archives_src = self.process_archives()
         # Extract all sources
         self.extract_sources(main_src, archives_src)
+        # Download and process any auto-generated source-tree archive for comparison
+        autogenerated_src = self.process_autogenerated_source(self.autogenerated_tarball)
+        # Extract autogenerated source for comparison
+        if autogenerated_src:
+            autogenerated_src.extract(os.path.join(self.base_path, 'autogenerated-tmp'))
+            # Move the autogenerated source to a non-version-named directory for consistent diffs
+            call(f"mv autogenerated-tmp/{autogenerated_src.prefix} autogenerated", check=True, cwd=self.base_path)
+            call("diff -u -r --unidirectional-new-file ../autogenerated ./",
+                 logfile="archive.diff.in", check=False, cwd=os.path.join(self.base_path, main_src.prefix))
+            call("grep -A14 -E '^(diff|Only in)' archive.diff.in",
+                 logfile="archive.diff", check=False, cwd=os.getcwd())
+            call("rm archive.diff.in", check=False, cwd=os.getcwd())

autospec/util.py

@@ -150,8 +150,9 @@ def binary_in_path(binary):
 
 def write_out(filename, content, mode="w"):
     """File.write convenience wrapper."""
-    with open_auto(filename, mode) as require_f:
-        require_f.write(content)
+    if mode:
+        with open_auto(filename, mode) as require_f:
+            require_f.write(content)
 
 
 def open_auto(*args, **kwargs):

tests/test_buildreq.py

@@ -402,7 +402,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'setup.py'), 'w').close()
@@ -419,7 +418,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'CMakeLists.txt'), 'w').close()
@@ -436,7 +434,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'SConstruct'), 'w').close()
@@ -453,7 +450,6 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
-        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'meson.build'), 'w').close()
@@ -484,7 +480,6 @@ class TestBuildreq(unittest.TestCase):
         buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
-            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
             self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
@@ -517,7 +512,6 @@ class TestBuildreq(unittest.TestCase):
         m_open = mock_open(read_data=content)
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
-            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
             open(os.path.join(tmpd, 'pypi.json'), 'w').close()
@@ -543,7 +537,6 @@ class TestBuildreq(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
-            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
             open(os.path.join(tmpd, 'subdir', 'requires.txt'), 'w').close()
@@ -553,21 +546,6 @@ class TestBuildreq(unittest.TestCase):
         self.reqs.add_setup_py_requires.assert_called_once()
         self.reqs.grab_python_requirements.assert_called_once()
 
-    def test_scan_for_configure_ninja(self):
-        """
-        Test scan_for_configure when ninja is enabled.
-        """
-        conf = config.Config("")
-        conf.config_opts['use_ninja'] = True
-        with tempfile.TemporaryDirectory() as tmpd:
-            os.mkdir(os.path.join(tmpd, 'subdir'))
-            open(os.path.join(tmpd, 'setup.py'), 'w').close()
-
-            self.reqs.scan_for_configure(tmpd, "", conf)
-
-        self.assertEqual(self.reqs.buildreqs,
-                         set(['buildreq-distutils3', 'ninja']))
-
     def test_parse_cmake_pkg_check_modules(self):
         """
         Test parse_cmake to ensure accurate detection of versioned and