Add the Provide: pypi in most cases

Usually the pypi ecosystem will do the right thing and respond with a
name but in cases where it does not (bad network, bad json, message
without a name), still figure out a reasonable Provide to add.

Signed-off-by: William Douglas <william.douglas@intel.com>

autospec/autospec.py

@@ -277,6 +277,9 @@ def package(args, url, name, archives, workingdir):
             # specfile template is assumed "correct" and any failures need to be manually addressed
             break
         filemanager.load_specfile(specfile)
+        if 'license' in specfile.packages and not conf.config_opts['has_license']:
+            conf.config_opts['has_license'] = True
+            conf.rewrite_config_opts()
         specfile.write_spec()
         filemanager.newfiles_printed = 0
         mock_chroot = "/var/lib/mock/clear-{}/root/builddir/build/BUILDROOT/" \
@@ -297,6 +300,10 @@ def package(args, url, name, archives, workingdir):
         conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
         print_build_failed()
         sys.exit(1)
+    elif 'license' not in specfile.packages and conf.config_opts['has_license']:
+        print_fatal("package -license subpackage deleted")
+        conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
+        sys.exit(1)
     elif os.path.isfile("README.clear"):
         try:
             print("\nREADME.clear CONTENTS")

autospec/build.py

@@ -215,6 +215,8 @@ class Build(object):
             elif infiles == 1 and "not matching the package arch" not in line:
                 # exclude blank lines from consideration...
                 file = line.strip()
+                if file in ("/usr", "/usr/"):
+                    self.must_restart += 1
                 if file and file[0] == "/":
                     filemanager.push_file(file, content.name)
 

autospec/buildreq.py

@@ -709,6 +709,7 @@ class Requirements(object):
         """Use pypi for getting package requires and metadata."""
         # First look for a local override
         pypi_json = ""
+        pypi_name = pypidata.get_pypi_name(name)
         pypi_file = os.path.join(config.download_path, "pypi.json")
         if os.path.isfile(pypi_file):
             with open(pypi_file, "r") as pfile:
@@ -716,16 +717,20 @@ class Requirements(object):
         else:
             # Try and grab the pypi details for the package
             if config.alias:
-                name = config.alias
-            pypi_name = pypidata.get_pypi_name(name)
+                pypi_name = config.alias
+            pypi_name = pypidata.get_pypi_name(pypi_name)
             pypi_json = pypidata.get_pypi_metadata(pypi_name)
-        if pypi_json:
+        if not pypi_json:
+            self.pypi_provides = pypi_name
+        else:
             try:
                 package_pypi = json.loads(pypi_json)
             except json.JSONDecodeError:
-                package_pypi = {}
+                package_pypi = {"name": pypi_name}
             if package_pypi.get("name"):
                 self.pypi_provides = package_pypi["name"]
+            else:
+                self.pypi_provides = pypi_name
             if package_pypi.get("requires"):
                 for pkg in package_pypi["requires"]:
                     self.add_requires(f"pypi({pkg})", config.os_packages, override=True, subpkg="python3")

autospec/config.py

@@ -193,6 +193,7 @@ class Config(object):
             "no_glob": "Do not use the replacement pattern for file matching",
             "allow_exe": "Allow Windows executables (*.exe, *.dll) to be packaged",
             "use_ninja": "Use ninja build files",
+            "has_license": "Require license subpackage for successful build",
         }
         # simple_pattern_pkgconfig patterns
         # contains patterns for parsing build.log for missing dependencies

autospec/count.py

@@ -71,14 +71,8 @@ def sanitize_counts():
     """Validate test counts are within sane bounds."""
     global total_tests
     global total_pass
-    global total_fail
-    global total_xfail
-    global total_skip
     global counted_tests
     global counted_pass
-    global counted_fail
-    global counted_xfail
-    global counted_skip
     if total_tests > 0 and total_pass == 0:
         total_pass = total_tests - total_fail - total_skip - total_xfail
 
@@ -170,7 +164,6 @@ def parse_log(log, pkgname=''):
     global total_fail
     global total_xfail
     global total_skip
-    global counted_tests
     global counted_pass
     global counted_fail
     global counted_xfail

autospec/license.py

@@ -55,8 +55,6 @@ def add_license(lic, translations, blacklist):
     presence in the blacklist. Returns False if no license were added, True
     otherwise.
     """
-    global licenses
-    global license_files
     lic = lic.strip().strip(',')
     result = False
 
@@ -208,9 +206,6 @@ def scan_for_licenses(srcdir, config, pkg_name):
 
 def load_specfile(specfile):
     """Get licenses from the specfile content."""
-    global licenses
-    global license_files
-    global hashes
     specfile.licenses = licenses if licenses else [default_license]
     specfile.license_files = sorted(license_files)
     specfile.hashes = hashes

autospec/pkg_integrity.py

@@ -96,7 +96,7 @@ class GPGCli(object):
         util.write_out(os.path.join(_gpghome, 'gpg.conf'), GNUPGCONF)
         if pubkey is not None:
             args = self.args + ['--import', pubkey]
-            output, err, code = self.exec_cmd(args)
+            _, err, code = self.exec_cmd(args)
             if code == -9:
                 raise Exception('Command {} timeout after {} seconds'.format(' '.join(args), CMD_TIMEOUT))
             elif code != 0:
@@ -212,16 +212,15 @@ class Verifier(object):
     @staticmethod
     def calc_sum(filepath, digest_algo):
         """Use digest_algo to calculate block sum of a file."""
-        BLOCK_SIZE = 4096
+        block_size = 4096
         with open(filepath, 'rb') as fp:
             digest = digest_algo()
-            for block in iter(lambda: fp.read(BLOCK_SIZE), b''):
+            for block in iter(lambda: fp.read(block_size), b''):
                 digest.update(block)
             return digest.hexdigest()
 
     def print_result(self, result, err_msg=''):
         """Display verification results."""
-        global EMAIL
         package_name = ''
         if self.url is not None:
             package_name = os.path.basename(self.url)

autospec/pypidata.py

@@ -27,6 +27,15 @@ def pkg_search(name):
         return False
 
 
+def fixup_pypi_prefix(name):
+    """Try and chop off the 'pypi-' or 'python-' prefix for names."""
+    name = name.lower().replace('-', '_')
+    for prefix in ["pypi_", "python_"]:
+        if name.startswith(prefix):
+            name = name[len(prefix):]
+    return name
+
+
 def get_pypi_name(name, miss=False):
     """Try and verify the pypi name for a given package name."""
     # normalize the name for matching as pypi is case insensitve for search
@@ -35,11 +44,9 @@ def get_pypi_name(name, miss=False):
     if pkg_search(name):
         return name
     # Maybe we have a prefix
-    for prefix in ["pypi_", "python_"]:
-        if name.startswith(prefix):
-            name = name[len(prefix):]
-            if pkg_search(name):
-                return name
+    name = fixup_pypi_prefix(name)
+    if pkg_search(name):
+        return name
     # Some cases where search fails (Sphinx)
     # Just try the name we were given
     if miss:

autospec/specfiles.py

@@ -37,9 +37,9 @@ AVX512_CFLAGS = "-march=x86-64-v4 -mprefer-vector-width=512"
 AVX512_FCFLAGS = "-march=x86-64-v4 -mprefer-vector-width=256"
 AVX512_LCFLAGS = "-march=x86-64-v4"
 AVX512_LFLAGS = "-Wl,-z,x86-64-v4"
-APX_CFLAGS = "-march=x86-64-v3 -mapxf -mavx10.1"
-APX_LCFLAGS = "-march=x86-64-v3"
-APX_LFLAGS = "-Wl,-z,x86-64-v3"
+APX_CFLAGS = "-march=x86-64-v4 -mapxf"
+APX_LCFLAGS = "-march=x86-64-v4"
+APX_LFLAGS = "-Wl,-z,x86-64-v4"
 
 
 class Specfile(object):
@@ -1166,7 +1166,7 @@ class Specfile(object):
             self.write_build_prepend()
             self._write_strip("GOAMD64=v3")
             self._write_strip(f'CFLAGS="$CLEAR_INTERMEDIATE_CFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
-            self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {AVX2_CFLAGS} {AVX2_LFLAGS} "')
+            self._write_strip(f'CXXFLAGS="$CLEAR_INTERMEDIATE_CXXFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'FFLAGS="$CLEAR_INTERMEDIATE_FFLAGS {APX_CFLAGS} {APX_LFLAGS} "')
             self._write_strip(f'FCFLAGS="$CLEAR_INTERMEDIATE_FCFLAGS {APX_CFLAGS} "')
             self._write_strip(f'LDFLAGS="$CLEAR_INTERMEDIATE_LDFLAGS {APX_LCFLAGS} "')

autospec/util.py

@@ -156,7 +156,6 @@ def _file_write(self, s):
 
 def translate(package):
     """Convert terms to their alternate definition."""
-    global dictionary
     for item in dictionary:
         if item.startswith(package + "="):
             return item.split("=")[1]

tests/test_buildreq.py

@@ -499,6 +499,95 @@ class TestBuildreq(unittest.TestCase):
         self.assertEqual(self.reqs.requires['python3'], pypi_requires)
         self.assertEqual(ssummary, summary)
 
+    def test_scan_for_configure_pypi_no_name_in_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test connecting to pypi but not getting a name back.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "pypi-name"
+        content = json.dumps({})
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_no_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test being unable to connect to pypi.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "pypi-name"
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value="")
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_bad_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test being given bad json data.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        orig_json_loads = buildreq.json.loads
+        name = "pypi-name"
+        content = json.dumps({})
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        buildreq.json.loads = MagicMock(side_effect=json.JSONDecodeError("", "", 0))
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+        buildreq.json.loads = orig_json_loads
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
     def test_scan_for_configure_pypi_override(self):
         """
         Test scan_for_configure when distutils is being used for the build