Add cargo vendor drop in style file

Add support for cargo_vendors which should be pretty much the output
of a 'cargo vendor' run. This will be appended to any existing
.cargo/config.toml file in the sources. It is intended to mostly come
from the vendor.py in common but useful for other builds as well which
don't need to enable cargo_vendor in options.conf.

Signed-off-by: William Douglas <william.douglas@intel.com>

README.rst

@@ -268,6 +268,12 @@ prep_prepend
   resulting ``.spec``, and is used for situations where fine-grained
   control is required.
 
+copy_prepend
+  Additional actions that should take place directly before the source
+  directory is copied for other builds (32bit, avx2, etc). This will be
+  placed in the resulting ``.spec``, and is used for situations where
+  fine-grained control is required.
+
 build_prepend
   Additional actions that should take place after ``%build`` and before
   the ``%configure`` macro or equivalent (``%cmake``, etc.). If autospec

autospec/check.py

@@ -97,17 +97,23 @@ def scan_for_tests(src_dir, config, requirements, content):
     }
     if config.config_opts.get('32bit'):
         testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../build32/clr-build32;\n" + cmake_check + " || :"
         testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
     if config.config_opts.get('use_avx2'):
         testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx2/clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx2;\n" + meson_check + " || :"
     if config.config_opts.get('use_avx512'):
         testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
-        testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx512/clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx512;\n" + meson_check + " || :"
+    if config.config_opts.get('use_apx'):
+        testsuites["makecheck"] += "\ncd ../buildapx;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildapx/clr-build-apx;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildapx;\n" + meson_check + " || :"
     if config.config_opts.get('openmpi'):
         testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
-        testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi
+        testsuites["cmake"] += "\ncd ../../build-openmpi/clr-build-openmpi;\n" + cmake_check_openmpi
 
     files = os.listdir(src_dir)
 

autospec/config.py

@@ -68,6 +68,7 @@ class Config(object):
 
     def __init__(self, download_path):
         """Initialize Default configuration settings."""
+        self.cargo_vendors = ""
         self.content = None  # hack to avoid circular init dependency
         self.extra_configure = ""
         self.extra_configure32 = ""
@@ -91,6 +92,7 @@ class Config(object):
         self.install_macro = "%make_install"
         self.disable_static = "--disable-static"
         self.prep_prepend = []
+        self.copy_prepend = []
         self.build_prepend = []
         self.build_prepend_once = []
         self.build_append = []
@@ -267,6 +269,7 @@ class Config(object):
             (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC \(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'),
             (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None),
             (r"Checking for (.*?)\.\.\.no", 0, None),
+            (r"checking for (.*?) \(using pkg-config\)\.\.\. no", 0, None),
             (r"Checking for (.*?)\s*: not found", 0, None),
             (r"Checking for (.*?)\s>=.*\s*: not found", 0, None),
             (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None),
@@ -289,6 +292,7 @@ class Config(object):
             (r"No rule to make target `(.*)',", 0, None),
             (r"Package (.*) was not found in the pkg-config search path.", 0, 'pkgconfig'),
             (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'),
+            (r"The file `([a-zA-Z0-9\-:]*)\.pc` needs to be installed and the PKG_CONFIG_PATH environment variable must contain its parent directory\.", 0, 'pkgconfig'),
             (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'),
             (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'),
             (r"Program (.*) found: NO", 0, None),
@@ -435,20 +439,24 @@ class Config(object):
         # next the options
         config_f['autospec'] = {}
         for fname, comment in sorted(self.config_options.items()):
+            fpath = os.path.join(self.download_path, fname)
             config_f.set('autospec', '# {}'.format(comment))
-            if os.path.exists(fname):
+            if os.path.isfile(fpath):
                 config_f['autospec'][fname] = 'true'
-                os.remove(fname)
+                os.remove(fpath)
             else:
                 config_f['autospec'][fname] = 'false'
 
         # default lto to true for new things
         config_f['autospec']['use_lto'] = 'true'
+        # default autoupdate to true for new things
+        config_f['autospec']['autoupdate'] = 'true'
 
         # renamed options need special care
-        if os.path.exists("skip_test_suite"):
+        skip_path = os.path.join(self.download_path, "skip_test_suite")
+        if os.path.exists(skip_path):
             config_f['autospec']['skip_tests'] = 'true'
-            os.remove("skip_test_suite")
+            os.remove(skip_path)
         self.write_config(config_f)
 
     def create_buildreq_cache(self, version, buildreqs_cache):
@@ -984,6 +992,11 @@ class Config(object):
                     if not license.add_license(word, self.license_translations, self.license_blacklist):
                         print_warning("{}: blacklisted license {} ignored.".format(self.content.name + ".license", word))
 
+        # cargo_vendors is the output of 'cargo vendor' and should be read as is
+        content = self.read_file(os.path.join(self.download_path, "cargo_vendors"), track=True)
+        if content:
+            self.cargo_vendors = "".join(content)
+
         if self.config_opts['use_clang']:
             self.config_opts['funroll-loops'] = False
             requirements.add_buildreq("llvm")
@@ -1002,6 +1015,7 @@ class Config(object):
             requirements.add_buildreq("openssh")
 
         self.prep_prepend = self.read_script_file(os.path.join(self.download_path, "prep_prepend"))
+        self.copy_prepend = self.read_script_file(os.path.join(self.download_path, "copy_prepend"))
         if os.path.isfile(os.path.join(self.download_path, "prep_append")):
             os.rename(os.path.join(self.download_path, "prep_append"), os.path.join(self.download_path, "build_prepend"))
         self.make_prepend = self.read_script_file(os.path.join(self.download_path, "make_prepend"))

autospec/license.py

@@ -178,7 +178,7 @@ def scan_for_licenses(srcdir, config, pkg_name):
             # seen in the DPDK 20.11.3 tree, where the `LICENSES` directory is
             # named `license` instead.
             dirbase = os.path.basename(dirpath)
-            if re.search(r'^(LICENSES|licenses?)$', dirbase) and re.search(r'\.txt$', name):
+            if re.search(r'^(LICENSES|licenses?|licensing)$', dirbase) and re.search(r'\.txt$', name):
                 license_from_copying_hash(os.path.join(dirpath, name),
                                           srcdir, config, pkg_name)
 

autospec/pkg_integrity.py

@@ -532,6 +532,7 @@ VERIFIER_TYPES = {
     '.bz2': GPGVerifier,
     '.xz': GPGVerifier,
     '.zip': GPGVerifier,
+    '.zst': GPGVerifier,
 }
 
 

autospec/specfiles.py

@@ -436,7 +436,7 @@ class Specfile(object):
             cmake_type = "Ninja"
         else:
             cmake_type = "Unix Makefiles"
-        cmake_string = f"cmake -G {cmake_type} -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
+        cmake_string = f"cmake -G '{cmake_type}' -DCMAKE_INSTALL_PREFIX=$MPI_ROOT -DCMAKE_INSTALL_SBINDIR=$MPI_BIN \\\n" \
                        '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \
                        '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \
                        '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n'
@@ -465,6 +465,11 @@ class Specfile(object):
                     extract_cmd = 'unzip -q {}'
                 if archive.endswith('.bz2') and not archive.endswith('.tar.bz2'):
                     extract_cmd = 'bzcat {0} > $(basename "{0}" .bz2)'
+                if archive.endswith('.zst'):
+                    if archive.endswith('.tar.zst'):
+                        extract_cmd = 'tar -I zstd xf {}'
+                    else:
+                        extract_cmd = 'zstd -dqc {0} > $(basename "{0}" .zst)'
                 self._write_strip('cd %{_builddir}')
                 archive_file = os.path.basename(archive)
                 if self.config.archive_details.get(archive + "prefix"):
@@ -504,38 +509,36 @@ class Specfile(object):
         self.apply_patches()
 
         # setup cargo.toml vendoring if needed
-        if self.config.config_opts['cargo_vendor']:
+        if self.config.cargo_vendors:
             if self.config.subdir:
                 self._write_strip("pushd " + self.config.subdir)
             self._write_strip("mkdir -p .cargo")
-            self._write_strip("echo '[source.crates-io]' >> .cargo/config.toml")
-            self._write_strip("""echo 'replace-with = "vendored-sources"' >> .cargo/config.toml""")
-            self._write_strip("echo '[source.vendored-sources]' >> .cargo/config.toml")
-            self._write_strip("""echo 'directory = "vendor"' >> .cargo/config.toml""")
+            self._write_strip(f"echo '{self.config.cargo_vendors}' >> .cargo/config.toml")
             if self.config.subdir:
                 self._write_strip("popd")
 
-        if self.config.default_pattern != 'cmake':
-            if self.config.config_opts['32bit']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_avx2']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_avx512']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['use_apx']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
-                self._write_strip("popd")
-            if self.config.config_opts['openmpi']:
-                self._write_strip("pushd ..")
-                self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
-                self._write_strip("popd")
+        self.write_copy_prepend()
+
+        if self.config.config_opts['32bit']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} build32".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildavx2".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildavx512".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['use_apx']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} buildapx".format(self.content.tarball_prefix))
+            self._write_strip("popd")
+        if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ..")
+            self._write_strip("cp -a {} build-openmpi".format(self.content.tarball_prefix))
+            self._write_strip("popd")
         self._write_strip("\n")
 
     def write_32bit_exports(self):
@@ -795,6 +798,14 @@ class Specfile(object):
                 self._write_strip("{}\n".format(line))
             self._write_strip("## prep_prepend end")
 
+    def write_copy_prepend(self):
+        """Write out any custom supplied commands prior to creating source copies for avx, etc builds."""
+        if self.config.copy_prepend:
+            self._write_strip("## copy_prepend content")
+            for line in self.config.copy_prepend:
+                self._write_strip("{}\n".format(line))
+            self._write_strip("## copy_prepend end")
+
     def write_build_prepend(self):
         """Write out any custom supplied commands at the start of the %build section and every build type."""
         if self.config.build_prepend:
@@ -910,13 +921,11 @@ class Specfile(object):
 
         self._write_strip("export GOAMD64=v2")
 
-        if self.config.subdir:
-            self._write_strip("pushd " + self.config.subdir)
-
         if self.config.config_opts['use_ninja'] and self.config.install_macro == '%make_install':
             self.config.install_macro = '%ninja_install'
 
         if self.config.config_opts['32bit']:
+            self._write_strip("pushd ../build32/" + self.config.subdir)
             self._write_strip("pushd clr-build32")
             self._write_strip("{}32 {} {}".format(self.config.install_macro,
                                                   self.config.extra_make_install,
@@ -934,30 +943,42 @@ class Specfile(object):
             self._write_strip("    popd")
             self._write_strip("fi")
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-avx2")
             self._write_strip("%s_v3 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
             self._write_strip("GOAMD64=v4")
             self._write_strip("pushd clr-build-avx512")
             self._write_strip("%s_v4 %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_apx']:
+            self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-apx")
             self._write_strip("%s_va %s || :\n" % (self.config.install_macro, self.config.extra_make_install))
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
             self._write_strip("GOAMD64=v3")
             self._write_strip("pushd clr-build-openmpi")
             self.write_install_openmpi()
             self._write_strip("popd")
+            self._write_strip("popd")
+
+        if self.config.subdir:
+            self._write_strip("pushd " + self.config.subdir)
 
         self._write_strip("GOAMD64=v2")
         self._write_strip("pushd clr-build")
@@ -1651,7 +1672,11 @@ class Specfile(object):
         self.write_make_line()
         self._write_strip("popd")
 
+        if self.config.subdir:
+            self._write_strip("popd")
+
         if self.config.config_opts['use_avx2']:
+            self._write_strip("pushd ../buildavx2/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-avx2")
             self._write_strip("pushd clr-build-avx2")
             self.write_build_prepend()
@@ -1664,8 +1689,10 @@ class Specfile(object):
             self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_avx512']:
+            self._write_strip("pushd ../buildavx512/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-avx512")
             self._write_strip("pushd clr-build-avx512")
             self.write_build_prepend()
@@ -1678,8 +1705,10 @@ class Specfile(object):
             self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['use_apx'] and not self.config.config_opts['use_clang']:
+            self._write_strip("pushd ../buildapx/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-apx")
             self._write_strip("pushd clr-build-apx")
             self.write_build_prepend()
@@ -1692,8 +1721,10 @@ class Specfile(object):
             self._write_strip(f"%cmake {self.config.cmake_srcdir} {self.extra_cmake} {cmake_type}")
             self.write_make_line()
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['32bit']:
+            self._write_strip("pushd ../build32/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build32")
             self._write_strip("pushd clr-build32")
             self.write_build_prepend()
@@ -1706,8 +1737,10 @@ class Specfile(object):
             self.write_make_line()
             self._write_strip("unset PKG_CONFIG_PATH")
             self._write_strip("popd")
+            self._write_strip("popd")
 
         if self.config.config_opts['openmpi']:
+            self._write_strip("pushd ../build-openmpi/" + self.config.subdir)
             self._write_strip("mkdir -p clr-build-openmpi")
             self._write_strip("pushd clr-build-openmpi")
             self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh")
@@ -1723,8 +1756,6 @@ class Specfile(object):
             self.write_make_line()
             self._write_strip("module unload openmpi")
             self._write_strip("popd")
-
-        if self.config.subdir:
             self._write_strip("popd")
 
         self._write_strip("\n")

autospec/tarball.py

@@ -26,6 +26,7 @@ import tarfile
 import zipfile
 
 import download
+import zstandard as zstd
 from util import do_regex, get_sha1sum, print_fatal, write_out
 
 
@@ -53,6 +54,8 @@ class Source():
             self.type = 'zip'
         elif self.url.lower().endswith(('.bz2')) and not self.url.lower().endswith(('.tar.bz2')):
             self.type = 'bz2'
+        elif self.url.lower().endswith('.zst'):
+            self.type = 'zst'
         else:
             self.type = 'tar'
 
@@ -81,6 +84,16 @@ class Source():
             print_fatal("Not a valid tar file.")
             sys.exit(1)
 
+    def set_zst_prefix(self):
+        """Determine prefix folder name of tar.zst file."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            lines = content.getnames()
+            if len(lines) == 0:
+                print_fatal("Zstd compressed tar file doesn't appear to have any content")
+                sys.exit(1)
+            elif len(lines) > 1:
+                self.prefix = os.path.commonpath(lines)
+
     def set_bz2_prefix(self):
         """No prefix for plain bz2 archives."""
 
@@ -128,6 +141,11 @@ class Source():
         with zipfile.ZipFile(self.path, 'r') as content:
             content.extractall(path=extraction_path)
 
+    def extract_zst(self, extraction_path):
+        """Extract zst in path."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            content.extractall(path=extraction_path)
+
 
 def convert_version(ver_str, name):
     """Remove disallowed characters from the version."""