Make autoupdate default for new packages

Things should be autoupdating by default for new content at this point as being held back for no reason isn't helpful. Signed-off-by: William Douglas <william.douglas@intel.com>
Fix meson check builddir typo
2026-06-16 02:45:56 +00:00 · 2024-07-08 12:24:12 -07:00 · 2024-07-02 12:17:59 -07:00 · 2024-07-01 01:36:31 -07:00 · 2024-07-01 01:36:31 -07:00 · 2024-07-01 01:36:31 -07:00
7 changed files with 111 additions and 35 deletions
@@ -268,6 +268,12 @@ prep_prepend
  resulting ``.spec``, and is used for situations where fine-grained
  control is required.

+copy_prepend
+  Additional actions that should take place directly before the source
+  directory is copied for other builds (32bit, avx2, etc). This will be
+  placed in the resulting ``.spec``, and is used for situations where
+  fine-grained control is required.
+
 build_prepend
  Additional actions that should take place after ``%build`` and before
  the ``%configure`` macro or equivalent (``%cmake``, etc.). If autospec
@@ -97,17 +97,23 @@ def scan_for_tests(src_dir, config, requirements, content):
    }
    if config.config_opts.get('32bit'):
        testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../build32/clr-build32;\n" + cmake_check + " || :"
        testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
    if config.config_opts.get('use_avx2'):
        testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx2/clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx2;\n" + meson_check + " || :"
    if config.config_opts.get('use_avx512'):
        testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx512/clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx512;\n" + meson_check + " || :"
+    if config.config_opts.get('use_apx'):
+        testsuites["makecheck"] += "\ncd ../buildapx;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildapx/clr-build-apx;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildapx;\n" + meson_check + " || :"
    if config.config_opts.get('openmpi'):
        testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
-        testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi
+        testsuites["cmake"] += "\ncd ../../build-openmpi/clr-build-openmpi;\n" + cmake_check_openmpi

    files = os.listdir(src_dir)

@@ -91,6 +91,7 @@ class Config(object):
        self.install_macro = "%make_install"
        self.disable_static = "--disable-static"
        self.prep_prepend = []
+        self.copy_prepend = []
        self.build_prepend = []
        self.build_prepend_once = []
        self.build_append = []
@@ -267,6 +268,7 @@ class Config(object):
            (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC \(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'),
            (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None),
            (r"Checking for (.*?)\.\.\.no", 0, None),
+            (r"checking for (.*?) \(using pkg-config\)\.\.\. no", 0, None),
            (r"Checking for (.*?)\s*: not found", 0, None),
            (r"Checking for (.*?)\s>=.*\s*: not found", 0, None),
            (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None),
@@ -289,6 +291,7 @@ class Config(object):
            (r"No rule to make target `(.*)',", 0, None),
            (r"Package (.*) was not found in the pkg-config search path.", 0, 'pkgconfig'),
            (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'),
+            (r"The file `([a-zA-Z0-9\-:]*)\.pc` needs to be installed and the PKG_CONFIG_PATH environment variable must contain its parent directory\.", 0, 'pkgconfig'),
            (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'),
            (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'),
            (r"Program (.*) found: NO", 0, None),
@@ -435,20 +438,24 @@ class Config(object):
        # next the options
        config_f['autospec'] = {}
        for fname, comment in sorted(self.config_options.items()):
+            fpath = os.path.join(self.download_path, fname)
            config_f.set('autospec', '# {}'.format(comment))
-            if os.path.exists(fname):
+            if os.path.isfile(fpath):
                config_f['autospec'][fname] = 'true'
-                os.remove(fname)
+                os.remove(fpath)
            else:
                config_f['autospec'][fname] = 'false'

        # default lto to true for new things
        config_f['autospec']['use_lto'] = 'true'
+        # default autoupdate to true for new things
+        config_f['autospec']['autoupdate'] = 'true'

        # renamed options need special care
-        if os.path.exists("skip_test_suite"):
+        skip_path = os.path.join(self.download_path, "skip_test_suite")
+        if os.path.exists(skip_path):
            config_f['autospec']['skip_tests'] = 'true'
-            os.remove("skip_test_suite")
+            os.remove(skip_path)
        self.write_config(config_f)

    def create_buildreq_cache(self, version, buildreqs_cache):
@@ -1002,6 +1009,7 @@ class Config(object):
            requirements.add_buildreq("openssh")

        self.prep_prepend = self.read_script_file(os.path.join(self.download_path, "prep_prepend"))
+        self.copy_prepend = self.read_script_file(os.path.join(self.download_path, "copy_prepend"))
        if os.path.isfile(os.path.join(self.download_path, "prep_append")):
            os.rename(os.path.join(self.download_path, "prep_append"), os.path.join(self.download_path, "build_prepend"))
        self.make_prepend = self.read_script_file(os.path.join(self.download_path, "make_prepend"))
@@ -178,7 +178,7 @@ def scan_for_licenses(srcdir, config, pkg_name):
            # seen in the DPDK 20.11.3 tree, where the `LICENSES` directory is
            # named `license` instead.
            dirbase = os.path.basename(dirpath)
-            if re.search(r'^(LICENSES|licenses?)$', dirbase) and re.search(r'\.txt$', name):
+            if re.search(r'^(LICENSES|licenses?|licensing)$', dirbase) and re.search(r'\.txt$', name):
                license_from_copying_hash(os.path.join(dirpath, name),
                                          srcdir, config, pkg_name)

@@ -532,6 +532,7 @@ VERIFIER_TYPES = {
    '.bz2': GPGVerifier,
    '.xz': GPGVerifier,
    '.zip': GPGVerifier,
+    '.zst': GPGVerifier,
 }


@@ -436,7 +436,7 @@ class Specfile(object):
            cmake_type = "Ninja"
        else:
            cmake_type = "Unix Makefiles"
-        cmake_string = f"cmake -G {cmake_type} -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
+        cmake_string = f"cmake -G '{cmake_type}' -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
                       '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \
                       '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \
                       '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n'
@@ -465,6 +465,11 @@ class Specfile(object):
                    extract_cmd = 'unzip -q {}'
                if archive.endswith('.bz2') and not archive.endswith('.tar.bz2'):
                    extract_cmd = 'bzcat {0} > $(basename "{0}" .bz2)'
+                if archive.endswith('.zst'):
+                    if archive.endswith('.tar.zst'):
+                        extract_cmd = 'tar -I zstd xf {}'
+                    else:
+                        extract_cmd = 'zstd -dqc {0} > $(basename "{0}" .zst)'
                self._write_strip('cd %{_builddir}')
                archive_file = os.path.basename(archive)
                if self.config.archive_details.get(archive + "prefix"):
@@ -515,27 +520,28 @@ class Specfile(object):
            if self.config.subdir:
                self._write_strip("popd")

-        if self.config.default_pattern != 'cmake':
-            if self.config.config_opts['32bit']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_avx2']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_avx512']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_apx']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['openmpi']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
-                self._write_strip("popd")
+        self.write_copy_prepend()
+
+        if self.config.config_opts['32bit']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_apx']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
+            self._write_strip("popd")
        self._write_strip("\n")

    def write_32bit_exports(self):
@@ -795,6 +801,14 @@ class Specfile(object):
                self._write_strip("{}\n".format(line))
            self._write_strip("## prep_prepend end")

+    def write_copy_prepend(self):
+        """Write out any custom supplied commands prior to creating source copies for avx, etc builds."""
+        if self.config.copy_prepend:
+            self._write_strip("## copy_prepend content")
+            for line in self.config.copy_prepend:
+                self._write_strip("{}\n".format(line))
+            self._write_strip("## copy_prepend end")
+
    def write_build_prepend(self):
        """Write out any custom supplied commands at the start of the %build section and every build type."""
        if self.config.build_prepend:
@@ -910,10 +924,11 @@ class Specfile(object):

        self._write_strip("export GOAMD64=v2")

-        if self.config.subdir:
-            self._write_strip("pushd " + self.config.subdir)
+        if self.config.config_opts['use_ninja'] and self.config.install_macro == '%make_install':
+            self.config.install_macro = '%ninja_install'

        if self.config.config_opts['32bit']:
+            self._write_strip("pushd ../build32/" + self.config.subdir)
            self._write_strip("pushd clr-build32")
            self._write_strip("{}32 {} {}".format(self.config.install_macro,
                                                  self.config.extra_make_install,
@@ -931,30 +946,42 @@ class Specfile(object):
            self._write_strip("    popd")
            self._write_strip("fi")
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
            self._write_strip("GOAMD64=v3")
            self._write_strip("pushd clr-build-avx2")
            self._write_strip("%s_v3 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
            self._write_strip("GOAMD64=v4")
            self._write_strip("pushd clr-build-avx512")
            self._write_strip("%s_v4 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['use_apx']:
+            self._write_strip("pushd ../buildapx/" + self.config.subdir)
            self._write_strip("GOAMD64=v3")
            self._write_strip("pushd clr-build-apx")
            self._write_strip("%s_va %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
            self._write_strip("GOAMD64=v3")
            self._write_strip("pushd clr-build-openmpi")
            self.write_install_openmpi()
            self._write_strip("popd")
+            self._write_strip("popd")
+
+        if self.config.subdir:
+            self._write_strip("pushd " + self.config.subdir)

        self._write_strip("GOAMD64=v2")
        self._write_strip("pushd clr-build")
@@ -1648,7 +1675,11 @@ class Specfile(object):
        self.write_make_line()
        self._write_strip("popd")

+        if self.config.subdir:
+            self._write_strip("popd")
+
        if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
            self._write_strip("mkdir -p clr-build-avx2")
            self._write_strip("pushd clr-build-avx2")
            self.write_build_prepend()
@@ -1661,8 +1692,10 @@ class Specfile(object):
            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
            self.write_make_line()
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
            self._write_strip("mkdir -p clr-build-avx512")
            self._write_strip("pushd clr-build-avx512")
            self.write_build_prepend()
@@ -1675,8 +1708,10 @@ class Specfile(object):
            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
            self.write_make_line()
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
+            self._write_strip("pushd ../buildapx/" + self.config.subdir)
            self._write_strip("mkdir -p clr-build-apx")
            self._write_strip("pushd clr-build-apx")
            self.write_build_prepend()
@@ -1689,8 +1724,10 @@ class Specfile(object):
            self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
            self.write_make_line()
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['32bit']:
+            self._write_strip("pushd ../build32/" + self.config.subdir)
            self._write_strip("mkdir -p clr-build32")
            self._write_strip("pushd clr-build32")
            self.write_build_prepend()
@@ -1703,8 +1740,10 @@ class Specfile(object):
            self.write_make_line()
            self._write_strip("unset PKG_CONFIG_PATH")
            self._write_strip("popd")
+            self._write_strip("popd")

        if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
            self._write_strip("mkdir -p clr-build-openmpi")
            self._write_strip("pushd clr-build-openmpi")
            self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh")
@@ -1720,8 +1759,6 @@ class Specfile(object):
            self.write_make_line()
            self._write_strip("module unload openmpi")
            self._write_strip("popd")
-
-        if self.config.subdir:
            self._write_strip("popd")

        self._write_strip("\n")
@@ -26,6 +26,7 @@ import tarfile
 import zipfile

 import download
+import zstandard as zstd
 from util import do_regex, get_sha1sum, print_fatal, write_out


@@ -53,6 +54,8 @@ class Source():
            self.type = 'zip'
        elif self.url.lower().endswith(('.bz2')) and not self.url.lower().endswith(('.tar.bz2')):
            self.type = 'bz2'
+        elif self.url.lower().endswith('.zst'):
+            self.type = 'zst'
        else:
            self.type = 'tar'

@@ -81,6 +84,16 @@ class Source():
            print_fatal("Not a valid tar file.")
            sys.exit(1)

+    def set_zst_prefix(self):
+        """Determine prefix folder name of tar.zst file."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            lines = content.getnames()
+            if len(lines) == 0:
+                print_fatal("Zstd compressed tar file doesn't appear to have any content")
+                sys.exit(1)
+            elif len(lines) > 1:
+                self.prefix = os.path.commonpath(lines)
+
    def set_bz2_prefix(self):
        """No prefix for plain bz2 archives."""

@@ -128,6 +141,11 @@ class Source():
        with zipfile.ZipFile(self.path, 'r') as content:
            content.extractall(path=extraction_path)

+    def extract_zst(self, extraction_path):
+        """Extract zst in path."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            content.extractall(path=extraction_path)
+

 def convert_version(ver_str, name):
    """Remove disallowed characters from the version."""
Author	SHA1	Message	Date
William Douglas	43d564b0b7	Make autoupdate default for new packages Things should be autoupdating by default for new content at this point as being held back for no reason isn't helpful. Signed-off-by: William Douglas <william.douglas@intel.com>	2024-07-08 12:24:12 -07:00
William Douglas	2659038eaa	Fix meson check builddir typo Signed-off-by: William Douglas <william.douglas@intel.com>	2024-07-02 12:17:59 -07:00
William Douglas	dc0ff31b43	Add support for a copy_prepend config file This patch has a few interrelated changes in it but primarily it is supporting a new copy_prepend configuration file for autospec. This is intended to support cases where changes should be made to the source directory prior to the source directory being copied for different builds (avx2, 32bit, etc). Also with this change some tweaks to how cmake builds are handled to be more aligned with other build systems. Primarily that the source directoy is now fully copied rather than just creating a cmake build directory per build in the same source directory. Finally check support has been updated to account for the new path and update support for meson and apx. Signed-off-by: William Douglas <william.douglas@intel.com>	2024-07-01 01:36:31 -07:00
William Douglas	3f1fa8e70b	Fix mpi cmake Unix Makefiles argument Signed-off-by: William Douglas <william.douglas@intel.com>	2024-07-01 01:36:31 -07:00
William Douglas	840d2ca0e2	Add new failed pattern detection Signed-off-by: William Douglas <william.douglas@intel.com>	2024-07-01 01:36:31 -07:00
Brett T. Warden	a5d3013703	Add pkgconfig detection pattern for rust dependencies	2024-06-25 13:13:03 -07:00
K1ngfish3r	381dfd88cc	make check!	2024-06-20 14:55:48 -07:00
K1ngfish3r	f483b68c90	make check?	2024-06-20 14:55:48 -07:00
K1ngfish3r	5d6bcfe2f7	zstd support	2024-06-20 14:55:48 -07:00
Brett T. Warden	f9eab4897e	Try a little bit harder to find licenses Look in directories named 'licensing'.	2024-06-18 13:57:32 -07:00
William Douglas	fbcebd0b3d	Add missing install handling for ninja For cmake builds, use ninja install if use_ninja is set. Signed-off-by: William Douglas <william.douglas@intel.com>	2024-05-30 01:15:04 -07:00