Add github url handling for tags with slashes

vectorscan uses 'vectorscan/' as a tag prefix which confuses our
version parsing (even more so with v being the tag prefix). Add
support for matching any '*/version' tag names as the top priority.

Signed-off-by: William Douglas <william.douglas@intel.com>

autospec/autospec.py

@@ -269,9 +269,13 @@ def package(args, url, name, archives, workingdir):
         pkg_integrity.check(url, conf, interactive=interactive_mode)
         pkg_integrity.load_specfile(specfile)
 
-    specfile.write_spec()
+    spec_type = specfile.write_spec()
+
     while 1:
         package.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, content, args.cleanup)
+        if spec_type == "template":
+            # specfile template is assumed "correct" and any failures need to be manually addressed
+            break
         filemanager.load_specfile(specfile)
         specfile.write_spec()
         filemanager.newfiles_printed = 0
@@ -304,7 +308,8 @@ def package(args, url, name, archives, workingdir):
         except Exception:
             pass
 
-    check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
+    if spec_type == "generate":
+        check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
 
     examine_abi(conf.download_path, content.name)
     if os.path.exists("/var/lib/rpm"):

autospec/specfiles.py

@@ -26,6 +26,8 @@ import types
 from collections import OrderedDict
 
 import git
+from jinja2 import Environment
+from jinja2.loaders import DictLoader
 from util import _file_write, open_auto
 
 AVX2_CFLAGS = "-march=x86-64-v3"
@@ -73,12 +75,42 @@ class Specfile(object):
 
     def write_spec(self):
         """Write spec file."""
-        self.specfile = open_auto("{}/{}.spec".format(self.config.download_path, self.name), "w")
+        spec_path = f"{os.path.join(self.config.download_path, self.name)}.spec"
+        self.specfile = open_auto(spec_path, "w")
         self.specfile.write_strip = types.MethodType(_file_write, self.specfile)
 
+        # last chance to sanitize url for template and build types
+        if self.config.urlban:
+            clean_url = re.sub(self.config.urlban, "localhost", self.url)
+            # Duplicate prefixes entry before we change the url
+            self.content.prefixes[clean_url] = self.content.prefixes.get(self.url)
+            self.url = clean_url
+
+        template_path = f"{spec_path}.template"
+
+        if os.path.isfile(template_path):
+            # make templates have a template build pattern
+            self.config.default_pattern = "template"
         # spec file comment header
         self.write_comment_header()
 
+        if os.path.isfile(template_path):
+            with open_auto(template_path) as tfile:
+                template_content = tfile.read()
+            template = Environment(loader=DictLoader({'spec': template_content})).get_template('spec')
+            kw = {
+                'package_name': self.name,
+                'package_version': self.version,
+                'package_url': self.url,
+                'package_release': self.release,
+            }
+            self.specfile.write(template.render(**kw))
+            self.specfile.write_strip('\n')
+            self.specfile.close()
+            # return specfile type built so autospec knows how to
+            # handle build results (template should only builds once)
+            return "template"
+
         if self.config.config_opts.get('keepstatic'):
             self._write("%define keepstatic 1\n")
 
@@ -110,6 +142,10 @@ class Specfile(object):
 
         self.specfile.close()
 
+        # return specfile type built so autospec knows how to
+        # handle build results (generate has multiple builds)
+        return "generate"
+
     def write_comment_header(self):
         """Write comment header to spec file."""
         self._write("#\n")
@@ -132,11 +168,6 @@ class Specfile(object):
 
     def write_nvr(self):
         """Write name, version, and release information."""
-        if self.config.urlban:
-            clean_url = re.sub(self.config.urlban, "localhost", self.url)
-            # Duplicate prefixes entry before we change the url
-            self.content.prefixes[clean_url] = self.content.prefixes.get(self.url)
-            self.url = clean_url
         self._write("Name     : {}\n".format(self.name))
         self._write("Version  : {}\n".format(self.version))
         self._write("Release  : {}\n".format(str(self.release)))
@@ -452,12 +483,16 @@ class Specfile(object):
             self._write_strip(f"gpg --homedir .gnupg --status-fd 1 --verify {self.config.signature_macro} %{{SOURCE0}} > gpg.status")
             self._write_strip(f"grep -E '^\\[GNUPG:\\] (GOODSIG|EXPKEYSIG) {self.keyid}' gpg.status")
         self.write_prep_prepend()
-        prefix = self.content.prefixes[self.url]
         if self.config.default_pattern == 'R':
             prefix = self.content.tarball_prefix
             self._write_strip("%setup -q -n " + prefix)
         else:
-            self._write_strip("%setup -q -n " + prefix)
+            prefix = self.content.prefixes[self.url]
+            if not prefix:
+                prefix = os.path.splitext(os.path.basename(self.url))[0]
+                self._write_strip("%setup -q -c -n " + prefix)
+            else:
+                self._write_strip("%setup -q -n " + prefix)
             for archive in self.config.sources["archive"]:
                 # Handle various archive types
                 extract_cmd = 'tar xf {}'

autospec/tarball.py

@@ -295,7 +295,8 @@ class Content():
         if "github.com" in self.url:
             # define regex accepted for valid packages, important for specific
             # patterns to come before general ones
-            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
+            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/.*/(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
                                r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",

autospec/util.py

@@ -90,6 +90,20 @@ def _process_line(line, prev_line, current_patch, reported_patches, error):
             _log_error("Compiler: " + m.group('error'))
             return True
 
+    if m := re.match(r'Could NOT find (?P<package>.*) .missing', line):
+        _log_error("CMake module " + m.group('package') + " not found")
+        return True
+    if m := re.match(r'Could not find a package configuration file provided by (?P<package>.*) with', line):
+        _log_error("CMake module " + m.group('package') + " not found")
+        return True
+    # Unable to find program 'gperf'
+    if m := re.match(r"Failed to find program ‘(?P<module>.*)’", line):
+        _log_error("Failed to find " + m.group('module'))
+        return True
+    if m := re.match(r"Failed to find ‘(?P<module>.*)’", line):
+        _log_error("Failed to find " + m.group('module'))
+        return True
+
     return False
 
 
@@ -98,7 +112,7 @@ def _process_build_log(filename):
         lines = lfile.readlines()
 
     prev_line = ''
-    current_patch = ''
+    current_patch = ['']
     reported_patches = {}
     error = False
     for line in lines:

tests/packageurls

@@ -1612,3 +1612,4 @@ http://sourceforge.net/projects/zsh/files/zsh/5.4.2/zsh-5.4.2.tar.gz,zsh,5.4.2
 https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3.11-pigeonhole-0.5.11.tar.gz,pigeonhole,0.5.11
 https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3-pigeonhole-0.5.20.tar.gz,pigeonhole,0.5.20
 https://www.ezix.org/software/files/lshw-B.02.19.2.tar.gz,lshw,02.19.2
+https://github.com/VectorCamp/vectorscan/archive/refs/tags/vectorscan/5.4.11.tar.gz,vectorscan,5.4.11

tests/test_specfile.py

@@ -47,7 +47,7 @@ class TestSpecfileWrite(unittest.TestCase):
         self.WRITES = self.WRITES[:4] + self.WRITES[6:]
         self.assertEqual(expect, self.WRITES)
 
-    def test_write_nvr_no_urlban(self):
+    def test_write_nvr(self):
         """
         test Specfile.write_nvr with no urlban set
         """
@@ -59,19 +59,6 @@ class TestSpecfileWrite(unittest.TestCase):
                   "Source0  : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n"]
         self.assertEqual(expect, self.WRITES)
 
-    def test_write_nvr_urlban(self):
-        """
-        test Specfile.write_nvr with urlban set
-        """
-        self.specfile.config.urlban = "www.testpkg.com"
-        self.specfile.write_nvr()
-        expect = ["Name     : pkg\n",
-                  "Version  : 1.0\n",
-                  "Release  : 2\n",
-                  "URL      : http://localhost/testpkg/pkg-1.0.tar.gz\n",
-                  "Source0  : http://localhost/testpkg/pkg-1.0.tar.gz\n"]
-        self.assertEqual(expect, self.WRITES)
-
     def test_write_sources(self):
         """
         test write_sources with all Specfile.sources set.