Add config for using ninja instead of make

Given more packages are using ninja as the build system of choice
instead of make, add flag to enable ninja usage.

Signed-off-by: William Douglas <william.douglas@intel.com>

autospec/buildreq.py

@@ -751,6 +751,10 @@ class Requirements(object):
         configure_ac_files = []
         qmake_profiles = []
         cmake_files = []
+
+        if config.config_opts['use_ninja']:
+            self.add_buildreq('ninja')
+
         for dirpath, _, files in os.walk(dirn):
             default_score = 2 if dirpath == dirn else 1
 

autospec/config.py

@@ -187,6 +187,7 @@ class Config(object):
             "server": "Package is only used by servers",
             "no_glob": "Do not use the replacement pattern for file matching",
             "allow_exe": "Allow Windows executables (*.exe, *.dll) to be packaged",
+            "use_ninja": "Use ninja build files",
         }
         # simple_pattern_pkgconfig patterns
         # contains patterns for parsing build.log for missing dependencies

autospec/git.py

@@ -113,7 +113,6 @@ def commit_to_git(config, name, success):
     call("git add whatrequires", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add description", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add attrs", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add archive.diff", check=False, stderr=subprocess.DEVNULL, cwd=path)
 
     # remove deprecated config files
     call("git rm make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)

autospec/specfiles.py

@@ -414,6 +414,8 @@ class Specfile(object):
             self._write_strip("## make_prepend end")
         if self.config.make_command:
             make = self.config.make_command
+        elif self.config.config_opts['use_ninja']:
+            make = "ninja"
         else:
             make = "make"
         if build32:
@@ -430,7 +432,11 @@ class Specfile(object):
 
     def write_cmake_line_openmpi(self):
         """Write cmake line (openmpi) to spec file."""
-        cmake_string = 'cmake -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n' \
+        if self.config.config_opts['use_ninja']:
+            cmake_type = "Ninja"
+        else:
+            cmake_type = "Unix Makefiles"
+        cmake_string = f"cmake -G {cmake_type} -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
                        '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \
                        '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \
                        '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n'
@@ -591,7 +597,7 @@ class Specfile(object):
             if self.config.config_opts['use_clang']:
                 flags.extend(["-O3"])
             else:
-                flags.extend(["-Ofast", "-fno-semantic-interposition", "-falign-functions=32"])
+                flags.extend(["-fno-semantic-interposition", "-falign-functions=32"])
         if not self.config.config_opts['full-debug-info'] and not self.config.config_opts['use_clang']:
             flags.extend(["-gno-variable-location-views", "-gno-column-info", "-femit-struct-debug-baseonly", "-fdebug-types-section", "-gz=zstd", "-g1"])
         if self.config.default_pattern != 'qmake' or self.config.default_pattern != 'qmake6':
@@ -1097,7 +1103,6 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC="gcc-14"')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1203,7 +1208,6 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC="gcc-14"')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1264,7 +1268,6 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1342,7 +1345,6 @@ class Specfile(object):
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1387,7 +1389,6 @@ class Specfile(object):
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1435,7 +1436,6 @@ class Specfile(object):
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1478,7 +1478,6 @@ class Specfile(object):
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self.write_build_prepend()
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1523,7 +1522,6 @@ class Specfile(object):
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("pushd ../buildapx/" + self.config.subdir)
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
@@ -1639,7 +1637,11 @@ class Specfile(object):
         self._write_strip("pushd clr-build")
         self.write_variables()
         self._write_strip("export GOAMD64=v2")
-        self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
+        if self.config.config_opts['use_ninja']:
+            cmake_type = "-G Ninja"
+        else:
+            cmake_type = "-G 'Unix Makefiles'"
+        self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
 
         self.write_profile_payload("cmake")
 
@@ -1656,7 +1658,7 @@ class Specfile(object):
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {AVX2_CFLAGS} "')
-            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
+            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
 
@@ -1670,7 +1672,7 @@ class Specfile(object):
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX512_CFLAGS} {AVX512_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {AVX512_CFLAGS} {AVX512_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {AVX512_CFLAGS} "')
-            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
+            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
 
@@ -1680,12 +1682,11 @@ class Specfile(object):
             self.write_build_prepend()
             self.write_variables()
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC=gcc-14')
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {APX_CFLAGS} "')
-            self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake))
+            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
 
@@ -1698,7 +1699,7 @@ class Specfile(object):
             self._write_strip("%cmake -DLIB_INSTALL_DIR:PATH=/usr/lib32 "
                               "-DCMAKE_INSTALL_LIBDIR=/usr/lib32 "
                               "-DLIB_SUFFIX=32 "
-                              "{} {} ".format(self.config.cmake_srcdir, self.extra_cmake))
+                              f"{self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("unset PKG_CONFIG_PATH")
             self._write_strip("popd")
@@ -1886,7 +1887,6 @@ class Specfile(object):
                 self._write_strip('ninja -v -C builddiravx512')
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC=gcc-14')
             if self.config.config_opts['pgo'] and self.config.profile_payload != "":
                 self._write_strip(f'CFLAGS="$CFLAGS_GENERATE {APX_CFLAGS} {APX_LFLAGS} "'
                                   f' CXXFLAGS="$CXXFLAGS_GENERATE {AVX2_CFLAGS} {AVX2_LFLAGS} "'
@@ -1957,7 +1957,6 @@ class Specfile(object):
             self._write_strip('DESTDIR=%{buildroot}-v4 ninja -C builddiravx512 install')
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
             self._write_strip("GOAMD64=v3")
-            self._write_strip('CC=gcc-14')
             self._write_strip('DESTDIR=%{buildroot}-va ninja -C builddirapx install')
 
         self._write_strip("GOAMD64=v2")

autospec/tarball.py

@@ -26,7 +26,7 @@ import tarfile
 import zipfile
 
 import download
-from util import call, do_regex, get_sha1sum, print_fatal, write_out
+from util import do_regex, get_sha1sum, print_fatal, write_out
 
 
 class Source():
@@ -184,7 +184,6 @@ class Content():
         self.prefixes = dict()
         self.config = config
         self.base_path = base_path
-        self.autogenerated_tarball = None
 
     def write_upstream(self, sha, tarfile, mode="w"):
         """Write the upstream hash to the upstream file."""
@@ -214,14 +213,6 @@ class Content():
         main_src = Source(url, '', src_path, self.config.default_pattern)
         return main_src
 
-    def process_autogenerated_source(self, url):
-        """Download any autogenerated source tarball for comparison."""
-        autogenerated_src = None
-        if url:
-            src_path = self.check_or_get_file(url, os.path.basename(url), mode="")
-            autogenerated_src = Source(url, '../autogenerated-tmp', src_path, self.config.default_pattern)
-        return autogenerated_src
-
     def print_header(self):
         """Print header for autospec run."""
         print("\n")
@@ -304,9 +295,6 @@ class Content():
                     name = re.sub(r"release-", '', name)
                     name = re.sub(r"\d*$", '', name)
                 self.rawname = name
-                # Identify the auto-generated tarball URL for comparison
-                if "/releases/download/" in self.url:
-                    self.autogenerated_tarball = "https://github.com/" + match.group(1).strip() + "/" + self.repo + "/archive/refs/tags/" + match.group(3).strip() + ".tar.gz"
                 version = match.group(3).replace(name, '')
                 if "/archive/" not in self.url:
                     version = re.sub(r"^[-_.a-zA-Z]+", "", version)
@@ -439,15 +427,3 @@ class Content():
         archives_src = self.process_archives()
         # Extract all sources
         self.extract_sources(main_src, archives_src)
-        # Download and process any auto-generated source-tree archive for comparison
-        autogenerated_src = self.process_autogenerated_source(self.autogenerated_tarball)
-        # Extract autogenerated source for comparison
-        if autogenerated_src:
-            autogenerated_src.extract(os.path.join(self.base_path, 'autogenerated-tmp'))
-            # Move the autogenerated source to a non-version-named directory for consistent diffs
-            call(f"mv autogenerated-tmp/{autogenerated_src.prefix} autogenerated", check=True, cwd=self.base_path)
-            call("diff -u -r --unidirectional-new-file ../autogenerated ./",
-                 logfile="archive.diff.in", check=False, cwd=os.path.join(self.base_path, main_src.prefix))
-            call("grep -A14 -E '^(diff|Only in)' archive.diff.in",
-                 logfile="archive.diff", check=False, cwd=os.getcwd())
-            call("rm archive.diff.in", check=False, cwd=os.getcwd())

autospec/util.py

@@ -150,9 +150,8 @@ def binary_in_path(binary):
 
 def write_out(filename, content, mode="w"):
     """File.write convenience wrapper."""
-    if mode:
-        with open_auto(filename, mode) as require_f:
-            require_f.write(content)
+    with open_auto(filename, mode) as require_f:
+        require_f.write(content)
 
 
 def open_auto(*args, **kwargs):

tests/test_buildreq.py

@@ -402,6 +402,7 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'setup.py'), 'w').close()
@@ -418,6 +419,7 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'CMakeLists.txt'), 'w').close()
@@ -434,6 +436,7 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'SConstruct'), 'w').close()
@@ -450,6 +453,7 @@ class TestBuildreq(unittest.TestCase):
         should be sufficient.
         """
         conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
         with tempfile.TemporaryDirectory() as tmpd:
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'meson.build'), 'w').close()
@@ -480,6 +484,7 @@ class TestBuildreq(unittest.TestCase):
         buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
             self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
@@ -512,6 +517,7 @@ class TestBuildreq(unittest.TestCase):
         m_open = mock_open(read_data=content)
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
             open(os.path.join(tmpd, 'pypi.json'), 'w').close()
@@ -537,6 +543,7 @@ class TestBuildreq(unittest.TestCase):
 
         with tempfile.TemporaryDirectory() as tmpd:
             conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
             os.mkdir(os.path.join(tmpd, 'subdir'))
             open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
             open(os.path.join(tmpd, 'subdir', 'requires.txt'), 'w').close()
@@ -546,6 +553,21 @@ class TestBuildreq(unittest.TestCase):
         self.reqs.add_setup_py_requires.assert_called_once()
         self.reqs.grab_python_requirements.assert_called_once()
 
+    def test_scan_for_configure_ninja(self):
+        """
+        Test scan_for_configure when ninja is enabled.
+        """
+        conf = config.Config("")
+        conf.config_opts['use_ninja'] = True
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'setup.py'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-distutils3', 'ninja']))
+
     def test_parse_cmake_pkg_check_modules(self):
         """
         Test parse_cmake to ensure accurate detection of versioned and