From 54636b48dda877ece159652fe8caa557c61fb831 Mon Sep 17 00:00:00 2001 From: William Douglas Date: Thu, 19 Mar 2020 16:45:13 -0700 Subject: [PATCH] Refactor the config into a class The config module had a large amount of globals that were being touched across many modules that would import. This made changes to config very fragile as figuring out what would be modified in any given call chain was difficult to diagnose. It also made testing fragile as one would need to reset a given module's config import to the best of their knowledge before rerunning another test. To get away from that (and to try and reduce the number of globally modified variablies in autospec as a whole), refactor the config module to provide its state as part config class. The long running goal of changes like this is to better track what content can be updated by a particular function (if a function would have access to the config instance is now the hint rather than the config module getting imported). --- autospec/autospec.py | 63 +- autospec/build.py | 23 +- autospec/buildreq.py | 96 +- autospec/check.py | 21 +- autospec/commitmessage.py | 19 +- autospec/config.py | 1692 ++++++++++++++---------------- autospec/files.py | 10 +- autospec/git.py | 3 +- autospec/license.py | 24 +- autospec/pkg_integrity.py | 36 +- autospec/pkg_scan.py | 7 +- autospec/specdescription.py | 37 +- autospec/specfiles.py | 238 +++-- autospec/tarball.py | 23 +- tests/test_build.py | 98 +- tests/test_buildreq.py | 107 +- tests/test_check.py | 33 +- tests/test_commitmessage.py | 39 +- tests/test_files.py | 9 +- tests/test_infile_update_spec.py | 3 +- tests/test_license.py | 54 +- tests/test_pkg_integrity.py | 94 +- tests/test_specdescription.py | 10 +- tests/test_specfile.py | 21 +- tests/test_tarball.py | 12 +- 25 files changed, 1370 insertions(+), 1402 deletions(-) diff --git a/autospec/autospec.py b/autospec/autospec.py index 02b6082..1c07bc7 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -85,11 +85,11 @@ def check_requirements(use_git): sys.exit(1) -def load_specfile(specfile): +def load_specfile(conf, specfile): """Gather all information from static analysis into Specfile instance.""" - config.load_specfile(specfile) + conf.load_specfile(specfile) tarball.load_specfile(specfile) - specdescription.load_specfile(specfile) + specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ) license.load_specfile(specfile) buildreq.load_specfile(specfile) buildpattern.load_specfile(specfile) @@ -125,10 +125,10 @@ def save_mock_logs(path, iteration): os.rename(src, dest) -def write_prep(workingdir): +def write_prep(conf, workingdir): """Write metadata to the local workingdir when --prep-only is used.""" - if config.urlban: - used_url = re.sub(config.urlban, "localhost", tarball.url) + if conf.urlban: + used_url = re.sub(conf.urlban, "localhost", tarball.url) else: used_url = tarball.url @@ -239,6 +239,7 @@ def main(): def package(args, url, name, archives, workingdir, infile_dict): """Entry point for building a package with autospec.""" + conf = config.Config() check_requirements(args.git) build.setup_workingdir(workingdir) @@ -246,9 +247,9 @@ def package(args, url, name, archives, workingdir, infile_dict): # First, download the tarball, extract it and then do a set # of static analysis on the content of the tarball. # - filemanager = files.FileManager() - tarball.process(url, name, args.version, args.target, archives, filemanager) - config.create_versions(build.download_path, tarball.multi_version) + filemanager = files.FileManager(conf) + tarball.process(url, name, args.version, args.target, archives, filemanager, conf) + conf.create_versions(build.download_path, tarball.multi_version) # Search up one level from here to capture multiple versions _dir = tarball.path @@ -262,36 +263,36 @@ def package(args, url, name, archives, workingdir, infile_dict): except Exception: pass # Start one directory higher so we scan *all* versions for licenses - license.scan_for_licenses(os.path.dirname(_dir)) + license.scan_for_licenses(os.path.dirname(_dir), conf) exit(0) - config.setup_patterns() - config.config_file = args.config - config.parse_config_files(build.download_path, args.bump, filemanager, tarball.version) - config.setup_patterns(config.failed_pattern_dir) - config.parse_existing_spec(build.download_path, tarball.name) + conf.setup_patterns() + conf.config_file = args.config + conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version) + conf.setup_patterns(conf.failed_pattern_dir) + conf.parse_existing_spec(build.download_path, tarball.name) if args.prep_only: - write_prep(workingdir) + write_prep(conf, workingdir) exit(0) buildreq.set_build_req() - buildreq.scan_for_configure(_dir, tarball.name, build.download_path) - specdescription.scan_for_description(tarball.name, _dir) + buildreq.scan_for_configure(_dir, tarball.name, build.download_path, conf) + specdescription.scan_for_description(tarball.name, _dir, conf.license_translations, conf.license_blacklist) # Start one directory higher so we scan *all* versions for licenses - license.scan_for_licenses(os.path.dirname(_dir)) - commitmessage.scan_for_changes(build.download_path, _dir) + license.scan_for_licenses(os.path.dirname(_dir), conf) + commitmessage.scan_for_changes(build.download_path, _dir, conf.transforms) add_sources(build.download_path, archives) - check.scan_for_tests(_dir) + check.scan_for_tests(_dir, conf) # # Now, we have enough to write out a specfile, and try to build it. # We will then analyze the build result and learn information until the # package builds # - specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release) + specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release, conf) filemanager.load_specfile(specfile) - load_specfile(specfile) + load_specfile(conf, specfile) # # If infile is passed, parse it and overwrite the specfile configurations @@ -303,12 +304,12 @@ def package(args, url, name, archives, workingdir, infile_dict): if args.integrity: interactive_mode = not args.non_interactive - pkg_integrity.check(url, build.download_path, interactive=interactive_mode) + pkg_integrity.check(url, build.download_path, conf, interactive=interactive_mode) pkg_integrity.load_specfile(specfile) specfile.write_spec(build.download_path) while 1: - build.package(filemanager, args.mock_config, args.mock_opts, args.cleanup) + build.package(filemanager, args.mock_config, args.mock_opts, conf, args.cleanup) filemanager.load_specfile(specfile) specfile.write_spec(build.download_path) filemanager.newfiles_printed = 0 @@ -326,10 +327,10 @@ def package(args, url, name, archives, workingdir, infile_dict): save_mock_logs(build.download_path, build.round) - check.check_regression(build.download_path) + check.check_regression(build.download_path, conf.config_opts['skip_tests']) if build.success == 0: - config.create_buildreq_cache(build.download_path, tarball.version) + conf.create_buildreq_cache(build.download_path, tarball.version) print_fatal("Build failed, aborting") sys.exit(1) elif os.path.isfile("README.clear"): @@ -345,18 +346,18 @@ def package(args, url, name, archives, workingdir, infile_dict): examine_abi(build.download_path) if os.path.exists("/var/lib/rpm"): - pkg_scan.get_whatrequires(tarball.name) + pkg_scan.get_whatrequires(tarball.name, conf.yum_conf) write_out(build.download_path + "/release", tarball.release + "\n") # record logcheck output logcheck(build.download_path) - commitmessage.guess_commit_message(pkg_integrity.IMPORTED) - config.create_buildreq_cache(build.download_path, tarball.version) + commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf) + conf.create_buildreq_cache(build.download_path, tarball.version) if args.git: - git.commit_to_git(build.download_path) + git.commit_to_git(build.download_path, conf) else: print("To commit your changes, git add the relevant files and " "run 'git commit -F commitmsg'") diff --git a/autospec/build.py b/autospec/build.py index c63a7f5..d577c87 100644 --- a/autospec/build.py +++ b/autospec/build.py @@ -24,7 +24,6 @@ import re import shutil import buildreq -import config import tarball import util @@ -45,13 +44,13 @@ def setup_workingdir(workingdir): download_path = os.path.join(base_path, tarball.name) -def simple_pattern_pkgconfig(line, pattern, pkgconfig): +def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32): """Check for pkgconfig patterns and restart build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: - must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, cache=True) + must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True) def simple_pattern(line, pattern, req): @@ -97,7 +96,7 @@ def cleanup_req(s: str) -> str: return s -def failed_pattern(line, pattern, verbose, buildtool=None): +def failed_pattern(line, config, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about @@ -119,11 +118,11 @@ def failed_pattern(line, pattern, verbose, buildtool=None): if req: must_restart += buildreq.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': - must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) + must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 - buildreq.add_requires("R-" + s) + buildreq.add_requires("R-" + s, config.os_packages) elif buildtool == 'perl': s = s.replace('inc::', '') must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) @@ -167,7 +166,7 @@ def failed_pattern(line, pattern, verbose, buildtool=None): # Fallback to mvn-ARTIFACTID package name must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True) elif buildtool == 'catkin': - must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) + must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) must_restart += buildreq.add_buildreq(s, cache=True) except Exception: @@ -207,7 +206,7 @@ def check_for_warning_pattern(line): util.print_warning("Build log contains: {}".format(pat)) -def parse_build_results(filename, returncode, filemanager): +def parse_build_results(filename, returncode, filemanager, config): """Handle build log contents.""" global must_restart global success @@ -222,13 +221,13 @@ def parse_build_results(filename, returncode, filemanager): for line in loglines: for pat in config.pkgconfig_pats: - simple_pattern_pkgconfig(line, *pat) + simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit')) for pat in config.simple_pats: simple_pattern(line, *pat) for pat in config.failed_pats: - failed_pattern(line, *pat) + failed_pattern(line, config, *pat) check_for_warning_pattern(line) @@ -277,7 +276,7 @@ def get_mock_cmd(): return 'sudo /usr/bin/mock' -def package(filemanager, mockconfig, mockopts, cleanup=False): +def package(filemanager, mockconfig, mockopts, config, cleanup=False): """Run main package build routine.""" global round global uniqueext @@ -343,7 +342,7 @@ def package(filemanager, mockconfig, mockopts, cleanup=False): is_clean = parse_buildroot_log(download_path + "/results/root.log", ret) if is_clean: - parse_build_results(download_path + "/results/build.log", ret, filemanager) + parse_build_results(download_path + "/results/build.log", ret, filemanager, config) if filemanager.has_banned: util.print_fatal("Content in banned paths found, aborting build") exit(1) diff --git a/autospec/buildreq.py b/autospec/buildreq.py index 59298b1..9e8f90b 100644 --- a/autospec/buildreq.py +++ b/autospec/buildreq.py @@ -23,10 +23,8 @@ import ast import json import os import re -import subprocess import buildpattern -import config import pypidata import specdescription import toml @@ -83,8 +81,8 @@ def add_buildreq(req, cache=False): return new -def add_requires(req, override=False): - """Add req to the global requires set if it is present in buildreqs and os_packages and is not banned.""" +def add_requires(req, packages, override=False): + """Add req to the global requires set if it is present in buildreqs and packages and is not banned.""" global buildreqs global requires new = True @@ -96,7 +94,7 @@ def add_requires(req, override=False): # Try dashes instead of underscores as some ecosystems are inconsistent in their naming req2 = req.replace("_", "-") - if req not in buildreqs and req2 in config.os_packages and req2 not in requires and req2 not in banned_requires: + if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: # Since this is done for python add a buildreq just in case (might not be correct though) buildreqs.add(req2) requires.add(req2) @@ -108,13 +106,13 @@ def add_requires(req, override=False): req2 = req[0].lower() + req[1:] else: req2 = req[0].upper() + req[1:] - if req not in buildreqs and req2 in config.os_packages and req2 not in requires and req2 not in banned_requires: + if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: # Since this is done for python add a buildreq just in case (might not be correct though) buildreqs.add(req2) requires.add(req2) return True - if req not in buildreqs and req not in config.os_packages and not override: + if req not in buildreqs and req not in packages and not override: if req: print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req)) return False @@ -124,16 +122,16 @@ def add_requires(req, override=False): return new -def add_pkgconfig_buildreq(preq, cache=False): +def add_pkgconfig_buildreq(preq, conf32, cache=False): """Format preq as pkgconfig req and add to buildreqs.""" - if config.config_opts['32bit']: + if conf32: req = "pkgconfig(32" + preq + ")" add_buildreq(req, cache) req = "pkgconfig(" + preq + ")" return add_buildreq(req, cache) -def configure_ac_line(line): +def configure_ac_line(line, conf32): """Parse configure_ac line and add appropriate buildreqs.""" # print("----\n", line, "\n----") # ignore comments @@ -164,7 +162,7 @@ def configure_ac_line(line): if len(L) > 1: rqlist = L[1].strip() for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req) + add_pkgconfig_buildreq(req, conf32) # PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found) match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line) @@ -172,7 +170,7 @@ def configure_ac_line(line): L = match.group(1).split(",") rqlist = L[0].strip() for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req) + add_pkgconfig_buildreq(req, conf32) def is_number(num_str): @@ -269,7 +267,7 @@ def parse_modules_list(modules_string, is_cmake=False): return res -def parse_configure_ac(filename): +def parse_configure_ac(filename, conf32): """Parse the configure.ac file for build requirements.""" buf = "" depth = 0 @@ -287,14 +285,14 @@ def parse_configure_ac(filename): if c != "\n": buf += c if c == "\n" and depth == 0: - configure_ac_line(buf) + configure_ac_line(buf, conf32) buf = "" - configure_ac_line(buf) + configure_ac_line(buf, conf32) f.close() -def parse_cargo_toml(filename): +def parse_cargo_toml(filename, packages): """Update build requirements using Cargo.toml. Set the build requirements for building rust programs using cargo. @@ -310,7 +308,7 @@ def parse_cargo_toml(filename): return for cdep in cargo["dependencies"]: if add_buildreq(cdep): - add_requires(cdep) + add_requires(cdep, packages) def _get_desc_field(field, desc): @@ -402,7 +400,7 @@ def _get_r_provides(): return set(provides) -def parse_r_description(filename): +def parse_r_description(filename, packages): """Update build/runtime requirements according to the R package description.""" deps = [] with util.open_auto(filename, "r") as desc: @@ -417,9 +415,9 @@ def parse_r_description(filename): if dep in r_provides: continue pkg = 'R-' + dep - if pkg in config.os_packages: + if pkg in packages: add_buildreq(pkg) - add_requires(pkg) + add_requires(pkg, packages) else: print("CRAN package '{}' not found in os_packages, skipping".format(pkg)) @@ -463,7 +461,7 @@ def set_build_req(): add_buildreq("rustc") -def rakefile(filename): +def rakefile(filename, gems): """Scan Rakefile for build requirements.""" with util.open_auto(filename, "r") as f: lines = f.readlines() @@ -473,14 +471,14 @@ def rakefile(filename): match = pat.search(line) if match: s = match.group(1) - if s != "rubygems" and s in config.gems: - print("Rakefile-dep: " + config.gems[s]) - add_buildreq(config.gems[s]) + if s != "rubygems" and s in gems: + print("Rakefile-dep: " + gems[s]) + add_buildreq(gems[s]) else: print("Rakefile-new: rubygem-" + s) -def parse_cmake(filename): +def parse_cmake(filename, cmake_modules, conf32): """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for.""" findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I) pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I) @@ -495,7 +493,7 @@ def parse_cmake(filename): if match: module = match.group(1) try: - pkg = config.cmake_modules[module] + pkg = cmake_modules[module] add_buildreq(pkg) except Exception: pass @@ -516,10 +514,10 @@ def parse_cmake(filename): module = wordmatch.group(2) # We have a match, so strip out any version info for m in parse_modules_list(module, is_cmake=True): - add_pkgconfig_buildreq(m) + add_pkgconfig_buildreq(m, conf32) -def qmake_profile(filename): +def qmake_profile(filename, qt_modules): """Scan .pro file for build requirements.""" with util.open_auto(filename, "r") as f: lines = f.readlines() @@ -533,7 +531,7 @@ def qmake_profile(filename): for module in s.split(): module = re.sub('-private$', '', module) try: - pc = config.qt_modules[module] + pc = qt_modules[module] add_buildreq('pkgconfig({})'.format(pc)) except Exception: pass @@ -575,7 +573,7 @@ def clean_python_req(req, add_python=True): return ret -def grab_python_requirements(descfile): +def grab_python_requirements(descfile, packages): """Add python requirements from requirements.txt file.""" if "/demo/" in descfile: return @@ -609,7 +607,7 @@ def grab_python_requirements(descfile): continue if clean_python_req(line) == 'mock': continue - add_requires(clean_python_req(line)) + add_requires(clean_python_req(line), packages) def get_python_build_version_from_classifier(filename): @@ -627,7 +625,7 @@ def get_python_build_version_from_classifier(filename): return "distutils3" -def add_setup_py_requires(filename): +def add_setup_py_requires(filename, packages): """Detect build requirements listed in setup.py in the install_requires and setup_requires lists. Handles the following patterns: @@ -675,7 +673,7 @@ def add_setup_py_requires(filename): dep = clean_python_req(ast.literal_eval(item), False) add_buildreq(dep) if req: - add_requires(dep) + add_requires(dep, packages) except Exception: # do not fail, the line contained a variable and @@ -699,7 +697,7 @@ def add_setup_py_requires(filename): dep = clean_python_req(ast.literal_eval(line), False) add_buildreq(dep) if req: - add_requires(dep) + add_requires(dep, packages) except Exception: # Do not fail, just keep looking @@ -721,7 +719,7 @@ def add_setup_py_requires(filename): dep = clean_python_req(dep) add_buildreq(dep) if req: - add_requires(dep) + add_requires(dep, packages) except Exception: # do not fail, the line contained a variable and had to @@ -729,7 +727,7 @@ def add_setup_py_requires(filename): pass -def parse_catkin_deps(cmakelists_file): +def parse_catkin_deps(cmakelists_file, conf32): """Determine requirements for catkin packages.""" f = util.open_auto(cmakelists_file, "r") lines = f.readlines() @@ -746,7 +744,7 @@ def parse_catkin_deps(cmakelists_file): comp = match.group("comp") if comp: for curr in comp.split(" "): - add_pkgconfig_buildreq(curr) + add_pkgconfig_buildreq(curr, conf32) catkin = True @@ -767,7 +765,7 @@ def is_qmake_pro(f): return f.endswith(".pro") and not f.startswith(".") -def scan_for_configure(dirn, tname, dlpath): +def scan_for_configure(dirn, tname, dlpath, config): """Scan the package directory for build files to determine build pattern.""" global pypi_provides global pypi_requires @@ -789,7 +787,7 @@ def scan_for_configure(dirn, tname, dlpath): add_buildreq("buildreq-scons") elif buildpattern.default_pattern == "R": add_buildreq("buildreq-R") - parse_r_description(os.path.join(dirn, "DESCRIPTION")) + parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages) elif buildpattern.default_pattern == "phpize": add_buildreq("buildreq-php") elif buildpattern.default_pattern == "nginx": @@ -819,7 +817,7 @@ def scan_for_configure(dirn, tname, dlpath): pkg = "go-" + req[0].replace("/", "-") add_buildreq(pkg) if buildpattern.default_pattern == "godep": - add_requires(pkg) + add_requires(pkg, config.os_packages) if "CMakeLists.txt" in files and "configure.ac" not in files: add_buildreq("buildreq-cmake") @@ -827,7 +825,7 @@ def scan_for_configure(dirn, tname, dlpath): srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or "..")) if os.path.samefile(dirpath, srcdir): - parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt")) + parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit')) if "configure" in files and os.access(dirpath + '/configure', os.X_OK): buildpattern.set_build_pattern("configure", default_score) @@ -836,11 +834,11 @@ def scan_for_configure(dirn, tname, dlpath): buildpattern.set_build_pattern("qmake", default_score) if "requires.txt" in files: - grab_python_requirements(dirpath + '/requires.txt') + grab_python_requirements(dirpath + '/requires.txt', config.os_packages) if "setup.py" in files: add_buildreq("buildreq-distutils3") - add_setup_py_requires(dirpath + '/setup.py') + add_setup_py_requires(dirpath + '/setup.py', config.os_packages) python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py') buildpattern.set_build_pattern(python_pattern, default_score) @@ -853,7 +851,7 @@ def scan_for_configure(dirn, tname, dlpath): buildpattern.set_build_pattern("scons", default_score) if "requirements.txt" in files: - grab_python_requirements(dirpath + '/requirements.txt') + grab_python_requirements(dirpath + '/requirements.txt', config.os_packages) if "meson.build" in files: add_buildreq("buildreq-meson") @@ -865,13 +863,13 @@ def scan_for_configure(dirn, tname, dlpath): for name in files: if name.lower() == "cargo.toml" and dirpath == dirn: - parse_cargo_toml(os.path.join(dirpath, name)) + parse_cargo_toml(os.path.join(dirpath, name), config.os_packages) if name.lower().startswith("configure."): - parse_configure_ac(os.path.join(dirpath, name)) + parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit')) if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby": - rakefile(os.path.join(dirpath, name)) + rakefile(os.path.join(dirpath, name), config.gems) if name.endswith(".pro") and buildpattern.default_pattern == "qmake": - qmake_profile(os.path.join(dirpath, name)) + qmake_profile(os.path.join(dirpath, name), config.qt_modules) if name.lower() == "makefile": buildpattern.set_build_pattern("make", default_score) if name.lower() == "autogen.sh": @@ -880,7 +878,7 @@ def scan_for_configure(dirn, tname, dlpath): buildpattern.set_build_pattern("cmake", default_score) if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \ and buildpattern.default_pattern == "cmake": - parse_cmake(os.path.join(dirpath, name)) + parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit')) can_reconf = os.path.exists(os.path.join(dirn, "configure.ac")) if not can_reconf: diff --git a/autospec/check.py b/autospec/check.py index a14edf6..a1d355f 100644 --- a/autospec/check.py +++ b/autospec/check.py @@ -24,7 +24,6 @@ import re import buildpattern import buildreq -import config import count import tarball import util @@ -32,9 +31,9 @@ import util tests_config = "" -def check_regression(pkg_dir): +def check_regression(pkg_dir, skip_tests): """Check the build log for test regressions using the count module.""" - if config.config_opts['skip_tests']: + if skip_tests: return result = count.parse_log(os.path.join(pkg_dir, "results/build.log")) @@ -56,11 +55,11 @@ def check_regression(pkg_dir): util.write_out(os.path.join(pkg_dir, "testresults"), res_str) -def scan_for_tests(src_dir): +def scan_for_tests(src_dir, config): """Scan source directory for test files and set tests_config accordingly.""" global tests_config - if config.config_opts['skip_tests'] or tests_config: + if config.config_opts.get('skip_tests') or tests_config: return makeflags = "%{?_smp_mflags} " if config.parallel_build else "" @@ -71,7 +70,7 @@ def scan_for_tests(src_dir): cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \ "make test\nmodule unload openmpi" - if config.config_opts['allow_test_failures']: + if config.config_opts.get('allow_test_failures'): make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \ "make VERBOSE=1 V=1 {}check || :\nmodule unload openmpi".format(makeflags) cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \ @@ -80,7 +79,7 @@ def scan_for_tests(src_dir): perl_check = "make TEST_VERBOSE=1 test" setup_check = """PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") python setup.py test""" meson_check = "meson test -C builddir" - if config.config_opts['allow_test_failures']: + if config.config_opts.get('allow_test_failures'): make_check += " || :" cmake_check += " || :" perl_check += " || :" @@ -96,17 +95,17 @@ def scan_for_tests(src_dir): "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd", "meson": meson_check, } - if config.config_opts['32bit']: + if config.config_opts.get('32bit'): testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :" testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :" - if config.config_opts['use_avx2']: + if config.config_opts.get('use_avx2'): testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :" - if config.config_opts['use_avx512']: + if config.config_opts.get('use_avx512'): testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :" - if config.config_opts['openmpi']: + if config.config_opts.get('openmpi'): testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi diff --git a/autospec/commitmessage.py b/autospec/commitmessage.py index 632c5dd..05cc74c 100644 --- a/autospec/commitmessage.py +++ b/autospec/commitmessage.py @@ -29,25 +29,24 @@ import sys from subprocess import PIPE, run import build -import config import tarball import util -def scan_for_changes(download_path, directory): +def scan_for_changes(download_path, directory, transforms): """Scan for changelogs or news files in the file sources. Scan for changelogs or news files in the source code and copy them to download_path as their - `config.transform`ed name. The file with the transformed name will later be parsed to find the + `transform`ed name. The file with the transformed name will later be parsed to find the commit message. """ found = [] - interests = config.transforms.keys() + interests = transforms.keys() for dirpath, dirnames, files in os.walk(directory, topdown=False): hits = [x for x in files if x.lower() in interests and x.lower() not in found] for item in hits: source = os.path.join(dirpath, item) - target = os.path.join(download_path, config.transforms[item.lower()]) + target = os.path.join(download_path, transforms[item.lower()]) try: shutil.copy(source, target) os.chmod(target, 0o644) @@ -82,7 +81,7 @@ def find_in_line(pattern, line): return bool(re.search(pattern, line)) -def process_NEWS(newsfile): +def process_NEWS(newsfile, old_version): """Parse the newfile for relevent changes. Look for changes and CVE fixes relevant to current version update. This information is returned @@ -99,7 +98,7 @@ def process_NEWS(newsfile): success = False start_found = False - if config.old_version is None or config.old_version == tarball.version: + if old_version is None or old_version == tarball.version: # no version update, so no information to search for in newsfile return commitmessage, cves @@ -113,7 +112,7 @@ def process_NEWS(newsfile): # escape some values for use in regular expressions below escaped_curver = re.escape(tarball.version) - escaped_oldver = re.escape(config.old_version) + escaped_oldver = re.escape(old_version) escaped_tarname = re.escape(tarball.name) # these are patterns that define the beginning of a block of information @@ -219,7 +218,7 @@ def process_git(giturl, oldversion, newversion): return shortlog -def guess_commit_message(keyinfo): +def guess_commit_message(keyinfo, config): """Parse newsfile for a commit message. Try and find a sane commit message for the newsfile. The commit message defaults to the @@ -271,7 +270,7 @@ def guess_commit_message(keyinfo): newsfiles = ["NEWS", "ChangeLog"] for newsfile in newsfiles: # parse news files for relevant version updates and cve fixes - newcommitmessage, newcves = process_NEWS(newsfile) + newcommitmessage, newcves = process_NEWS(newsfile, config.old_version) commitmessage.extend(newcommitmessage) cves.update(newcves) diff --git a/autospec/config.py b/autospec/config.py index b400248..292d6ea 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -35,349 +35,6 @@ import tarball from util import call, print_warning, write_out from util import open_auto -extra_configure = "" -extra_configure32 = "" -extra_configure64 = "" -extra_configure_avx2 = "" -extra_configure_avx512 = "" -config_files = set() -parallel_build = " %{?_smp_mflags} " -urlban = "" -extra_make = "" -extra32_make = "" -extra_make_install = "" -extra_make32_install = "" -extra_cmake = "" -extra_cmake_openmpi = "" -cmake_srcdir = "" -subdir = "" -install_macro = "%make_install" -disable_static = "--disable-static" -prep_prepend = [] -build_prepend = [] -build_append = [] -make_prepend = [] -install_prepend = [] -install_append = [] -service_restart = [] -patches = [] -verpatches = OrderedDict() -extra_sources = [] -autoreconf = False -custom_desc = "" -custom_summ = "" -set_gopath = True - -license_fetch = None -license_show = None -git_uri = None -os_packages = set() -config_file = None -old_version = None -old_patches = list() -old_keyid = None -profile_payload = None -signature = None -yum_conf = None -failed_pattern_dir = None -alias = None - -failed_commands = {} -ignored_commands = {} -maven_jars = {} -gems = {} -license_hashes = {} -license_translations = {} -license_blacklist = {} -qt_modules = {} -cmake_modules = {} - -cves = [] - -conf_args_openmpi = '--program-prefix= --exec-prefix=$MPI_ROOT \\\n' \ - '--libdir=$MPI_LIB --bindir=$MPI_BIN --sbindir=$MPI_BIN --includedir=$MPI_INCLUDE \\\n' \ - '--datarootdir=$MPI_ROOT/share --mandir=$MPI_MAN -exec-prefix=$MPI_ROOT --sysconfdir=$MPI_SYSCONFIG \\\n' \ - '--build=x86_64-generic-linux-gnu --host=x86_64-generic-linux-gnu --target=x86_64-clr-linux-gnu ' - -# Keep track of the package versions -versions = OrderedDict() -# Only parse the versions file once, and save the result for later -parsed_versions = OrderedDict() - -# defines which files to rename and copy to autospec directory, -# used in commitmessage.py -transforms = { - 'changes': 'ChangeLog', - 'changelog.txt': 'ChangeLog', - 'changelog': 'ChangeLog', - 'change.log': 'ChangeLog', - 'ChangeLog.md': 'ChangeLog', - 'changes.rst': 'ChangeLog', - 'changes.txt': 'ChangeLog', - 'news': 'NEWS', - 'meson_options.txt': 'meson_options.txt' -} - -config_opts = {} -config_options = { - "broken_c++": "extend flags with '-std=gnu++98", - "use_lto": "configure build for lto", - "use_avx2": "configure build for avx2", - "use_avx512": "configure build for avx512", - "keepstatic": "do not remove static libraries", - "asneeded": "unset %build LD_AS_NEEDED variable", - "allow_test_failures": "allow package to build with test failures", - "skip_tests": "Do not run test suite", - "no_autostart": "do not require autostart subpackage", - "optimize_size": "optimize build for size over speed", - "funroll-loops": "optimize build for speed over size", - "fast-math": "pass -ffast-math to compiler", - "insecure_build": "set flags to smallest -02 flags possible", - "conservative_flags": "set conservative build flags", - "broken_parallel_build": "disable parallelization during build", - "pgo": "set profile for pgo", - "use_clang": "add clang flags", - "32bit": "build 32 bit libraries", - "nostrip": "disable stripping binaries", - "verify_required": "require package verification for build", - "security_sensitive": "set flags for security-sensitive builds", - "so_to_lib": "add .so files to the lib package instead of dev", - "dev_requires_extras": "dev package requires the extras to be installed", - "autoupdate": "this package is trusted enough to automatically update " - "(used by other tools)", - "compat": "this package is a library compatibility package and only " - "ships versioned library files", - "nodebug": "do not generate debuginfo for this package", - "openmpi": "configure build also for openmpi"} - -# simple_pattern_pkgconfig patterns -# contains patterns for parsing build.log for missing dependencies -pkgconfig_pats = [ - (r"which: no qmake", "Qt"), - (r"XInput2 extension not found", "xi"), - (r"checking for UDEV\.\.\. no", "udev"), - (r"checking for UDEV\.\.\. no", "libudev"), - (r"XMLLINT not set and xmllint not found in path", "libxml-2.0"), - (r"error\: xml2-config not found", "libxml-2.0"), - (r"error: must install xorg-macros", "xorg-macros")] - -# simple_pattern patterns -# contains patterns for parsing build.log for missing dependencies -simple_pats = [ - (r'warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/.*"', "docbook-xml"), - (r"gobject-introspection dependency was not found, gir cannot be generated.", "gobject-introspection-dev"), - (r"gobject-introspection dependency was not found, gir cannot be generated.", "glibc-bin"), - (r"Cannot find development files for any supported version of libnl", "libnl-dev"), - (r"/", "cmake"), - (r"\-\- Boost libraries:", "boost-dev"), - (r"XInput2 extension not found", "inputproto"), - (r"^WARNING: could not find 'runtest'$", "dejagnu"), - (r"^WARNING: could not find 'runtest'$", "expect"), - (r"^WARNING: could not find 'runtest'$", "tcl"), - (r"VignetteBuilder package required for checking but installed:", "R-knitr"), - (r"You must have XML::Parser installed", "perl(XML::Parser)"), - (r"checking for Apache .* module support", "httpd-dev"), - (r"checking for.*in -ljpeg... no", "libjpeg-turbo-dev"), - (r"fatal error\: zlib\.h\: No such file or directory", "zlib-dev"), - (r"\* tclsh failed", "tcl"), - (r"\/usr\/include\/python3\.[0-9]+m\/pyconfig.h", "python3-dev"), - (r"checking \"location of ncurses\.h file\"", "ncurses-dev"), - (r"Can't exec \"aclocal\"", "automake"), - (r"Can't exec \"aclocal\"", "libtool"), - (r"configure: error: no suitable Python interpreter found", "python3-dev"), - (r"Checking for header Python.h", "python3-dev"), - (r"configure: error: No curses header-files found", "ncurses-dev"), - (r" \/usr\/include\/python3\.", "python3-dev"), - (r"to compile python extensions", "python3-dev"), - (r"testing autoconf... not found", "autoconf"), - (r"configure\: error\: could not find Python headers", "python3-dev"), - (r"checking for libxml libraries", "libxml2-dev"), - (r"checking for slang.h... no", "slang-dev"), - (r"configure: error: no suitable Python interpreter found", "python3"), - (r"configure: error: pcre-config for libpcre not found", "pcre"), - (r"checking for OpenSSL", "openssl-dev"), - (r"Package systemd was not found in the pkg-config search path.", "systemd-dev"), - (r"Unable to find the requested Boost libraries.", "boost-dev"), - (r"libproc not found. Please configure without procps", "procps-ng-dev"), - (r"configure: error: glib2", "glib-dev"), - (r"C library 'efivar' not found", "efivar-dev"), - (r"Has header \"efi.h\": NO", "gnu-efi-dev"), - (r"ERROR: Could not execute Vala compiler", "vala"), - (r".*: error: HAVE_INTROSPECTION does not appear in AM_CONDITIONAL", 'gobject-introspection-dev')] - -# failed_pattern patterns -# contains patterns for parsing build.log for missing dependencies -failed_pats = [ - (r" ! ([a-zA-Z:]+) is not installed", 0, 'perl'), - (r" ([a-zA-Z]+\:\:[a-zA-Z]+) not installed", 1, None), - (r"(?:Could|Did) (?:NOT|not) find ([a-zA-Z0-9]+)", 0, None), - (r" ([a-zA-Z0-9\-]*\.m4) not found", 0, None), - (r" exec: ([a-zA-Z0-9\-]+): not found", 0, None), - (r"([a-zA-Z0-9\-\_\.]*)\: command not found", 1, None), - (r"([a-zA-Z\-]*) (?:validation )?tool not found or not executable", 0, None), - (r"([a-zA-Z\-]+) [0-9\.]+ is required to configure this module; " - r"please install it or upgrade your CPAN\/CPANPLUS shell.", 0, None), - (r"-- (.*) not found.", 1, None), - (r".* /usr/bin/([a-zA-Z0-9-_]*).*not found", 0, None), - (r".*\.go:.*cannot find package \"(.*)\" in any of:", 0, 'go'), - (r"/usr/bin/env\: (.*)\: No such file or directory", 0, None), - (r"/usr/bin/python.*\: No module named (.*)", 0, None), - (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:\/]+)", 0, 'ruby table'), - (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:]+) ", 0, 'ruby'), - (r"Add the installation prefix of \"(.*)\" to CMAKE_PREFIX_PATH", 0, None), - (r"By not providing \"([a-zA-Z0-9]+).cmake\" in CMAKE_MODULE_PATH this project", 0, None), - (r"C library '(.*)' not found", 0, None), - (r"CMake Error at cmake\/modules\/([a-zA-Z0-9]+).cmake", 0, None), - (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC " r"\(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'), - (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None), - (r"Checking for (.*?)\.\.\.no", 0, None), - (r"Checking for (.*?)\s*: not found", 0, None), - (r"Checking for (.*?)\s>=.*\s*: not found", 0, None), - (r"Could not find '([a-zA-Z0-9\-\_]*)' \([~<>=]+ ([0-9.]+).*\) among [0-9]+ total gem", 0, 'ruby'), - (r"Could not find gem '([a-zA-Z0-9\-\_]+) \([~<>=0-9\.\, ]+\) ruby'", 0, 'ruby'), - (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None), - (r"Download error on https://pypi.python.org/simple/([a-zA-Z0-9\-\._:]+)/", 0, 'pypi'), - (r"Downloading https?://.*\.python\.org/packages/.*/.?/([A-Za-z]*)/.*", 0, None), - (r"ERROR: Could not find a valid gem '([a-zA-Z0-9\-\_\:]*)' \([>=]+ ([0-9.]+).*\)", 0, 'ruby'), - (r"ERROR: dependencies ['‘]([a-zA-Z0-9\-\.]*)['’].* are not available for package ['‘].*['’]", 0, 'R'), - (r"ERROR: dependencies ['‘].*['’], ['‘]([a-zA-Z0-9\-\.]*)['’],.* are not available for package ['‘].*['’]", 0, 'R'), - (r"ERROR: dependencies.*['‘]([a-zA-Z0-9\-\.]*)['’] are not available for package ['‘].*['’]", 0, 'R'), - (r"ERROR: dependency ['‘]([a-zA-Z0-9\-\.]*)['’] is not available for package ['‘].*['’]", 0, 'R'), - (r"Error: Unable to find (.*)", 0, None), - (r"Error: package ['‘]([a-zA-Z0-9\-\.]*)['’] required by", 0, 'R'), - (r"Gem::LoadError: Could not find '([a-zA-Z0-9\-\_]*)'", 0, 'ruby'), - (r"ImportError:.* No module named '?([a-zA-Z0-9\-\._]+)'?", 0, 'pypi'), - (r"ImportError\: ([a-zA-Z]+) module missing", 0, None), - (r"ImportError\: (?:No module|cannot import) named? (.*)", 0, None), - (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:\/\_]+)", 0, 'ruby table'), - (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:]+)/.*", 0, 'ruby'), - (r"ModuleNotFoundError.*No module named (.*)", 0, None), - (r"Native dependency '(.*)' not found", 0, "pkgconfig"), - (r"No library found for -l([a-zA-Z\-])", 0, None), - (r"No (?:matching distribution|local packages or working download links) found for ([a-zA-Z0-9\-\.\_]+)", 0, 'pypi'), - (r"No package '([a-zA-Z0-9\-:]*)' found", 0, 'pkgconfig'), - (r"No rule to make target `(.*)',", 0, None), - (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'), - (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'), - (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'), - (r"Program (.*) found: NO", 0, None), - (r"Target '[a-zA-Z0-9\-]' can't be generated as '(.*)' could not be found", 0, None), - (r"Unable to `import (.*)`", 0, None), - (r"Unable to find '(.*)'", 0, None), - (r"Unknown packages? ['‘]([a-zA-Z0-9\-]*)['’].* in Rd xrefs", 0, 'R'), - (r"WARNING: [a-zA-Z\-\_]+ dependency on ([a-zA-Z0-9\-\_:]*) \([<>=~]+ ([0-9.]+).*\) .*", 0, 'ruby'), - (r"Warning: prerequisite ([a-zA-Z:]+) [0-9\.]+ not found.", 0, 'perl'), - (r"Warning\: no usable ([a-zA-Z0-9]+) found", 0, None), - (r"You need ([a-zA-Z0-9\-\_]*) to build this program.", 1, None), - (r"[Dd]ependency (.*) found: NO \(tried pkgconfig(?: and cmake)?\)", 0, 'pkgconfig'), - (r"[Dd]ependency (.*) found: NO", 0, None), - (r"[a-zA-Z0-9\-:]* is not installed: cannot load such file -- rdoc/([a-zA-Z0-9\-:]*)", 0, 'ruby'), - (r"\-\- Could NOT find ([a-zA-Z0-9]+)", 0, None), - (r"\/bin\/ld: cannot find (-l[a-zA-Z0-9\_]+)", 0, None), - (r"^.*By not providing \"Find(.*).cmake\" in CMAKE_MODULE_PATH this.*$", 0, None), - (r"^.*Could not find a package configuration file provided by \"(.*)\".*$", 0, None), - (r"^.*\"(.*)\" with any of the following names.*$", 0, None), - (r"[Cc]hecking for (.*) (?:support|development files|with pkg-config)?\.\.\. [Nn]o", 0, None), - (r"checking (.*?)\.\.\. no", 0, None), - (r"checking for (.*) in default path\.\.\. not found", 0, None), - (r"checking for (.*)... configure: error", 0, None), - (r"checking for (.*?)\.\.\. no", 0, None), - (r"checking for [a-zA-Z0-9\_\-]+ in (.*?)\.\.\. no", 0, None), - (r"checking for library containing (.*)... no", 0, None), - (r"checking for perl module ([a-zA-Z:]+) [0-9\.]+... no", 0, 'perl'), - (r"configure: error: (?:pkg-config missing|Unable to locate) (.*)", 0, None), - (r"configure: error: ([a-zA-Z0-9]+) (?:is required to build|not found)", 0, None), - (r"configure: error: Cannot find (.*)\. Make sure", 0, None), - (r"fatal error\: (.*)\: No such file or directory", 0, None), - (r"make: ([a-zA-Z0-9].+): Command not found", 0, None), - (r"meson\.build\:[\d]+\:[\d]+\: ERROR: C library \'(.*)\' not found", 0, None), - (r"there is no package called ['‘]([a-zA-Z0-9\-\.]*)['’]", 0, 'R'), - (r"unable to execute '([a-zA-Z\-]*)': No such file or directory", 0, None), - (r"warning: failed to load external entity " r"\"(/usr/share/sgml/docbook/xsl-stylesheets)/.*\"", 0, None), - (r"which\: no ([a-zA-Z\-]*) in \(", 0, None), - (r"you may need to install the ([a-zA-Z0-9_\-:\.]*) module", 0, 'perl'), -] - - -def get_metadata_conf(): - """Gather package metadata from the tarball module.""" - metadata = {} - metadata['name'] = tarball.name - if urlban: - metadata['url'] = re.sub(urlban, "localhost", tarball.url) - metadata['archives'] = re.sub(urlban, "localhost", " ".join(tarball.archives)) - else: - metadata['url'] = tarball.url - metadata['archives'] = " ".join(tarball.archives) - - metadata['giturl'] = tarball.giturl - metadata['domain'] = tarball.domain - - if alias: - metadata['alias'] = alias - else: - metadata['alias'] = "" - - return metadata - - -def create_conf(path): - """Create options.conf file and use deprecated configuration files or defaults to populate.""" - config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) - - # first the metadata - config_f['package'] = get_metadata_conf() - - # next the options - config_f['autospec'] = {} - for fname, comment in sorted(config_options.items()): - config_f.set('autospec', '# {}'.format(comment)) - if os.path.exists(fname): - config_f['autospec'][fname] = 'true' - os.remove(fname) - else: - config_f['autospec'][fname] = 'false' - - # default lto to true for new things - config_f['autospec']['use_lto'] = 'true' - - # renamed options need special care - if os.path.exists("skip_test_suite"): - config_f['autospec']['skip_tests'] = 'true' - os.remove("skip_test_suite") - write_config(config_f, path) - - -def create_buildreq_cache(path, version): - """Make the buildreq_cache file.""" - content = read_conf_file(os.path.join(path, "buildreq_cache")) - # don't create an empty cache file - if len(buildreq.buildreqs_cache) < 1: - try: - # file was possibly added to git so we should clean it up - os.unlink(content) - except Exception: - pass - return - if not content: - pkgs = sorted(buildreq.buildreqs_cache) - else: - pkgs = sorted(set(content[1:]).union(buildreq.buildreqs_cache)) - with open(os.path.join(path, 'buildreq_cache'), "w") as cachefile: - cachefile.write("\n".join([version] + pkgs)) - config_files.add('buildreq_cache') - - -def create_versions(path, versions): - """Make versions file.""" - with open(os.path.join(path, "versions"), 'w') as vfile: - for version in versions: - vfile.write(version) - if versions[version]: - vfile.write('\t' + versions[version]) - vfile.write('\n') - config_files.add("versions") - def write_config(config_f, path): """Write the config_f to configfile.""" @@ -385,104 +42,6 @@ def write_config(config_f, path): config_f.write(configfile) -def read_config_opts(path): - """Read config options from path/options.conf.""" - global config_opts - global transforms - global alias - - opts_path = os.path.join(path, 'options.conf') - if not os.path.exists(opts_path): - create_conf(path) - - config_f = configparser.ConfigParser(interpolation=None) - config_f.read(opts_path) - if "autospec" not in config_f.sections(): - print("Missing autospec section in options.conf") - sys.exit(1) - - if 'package' in config_f.sections() and config_f['package'].get('alias'): - alias = config_f['package']['alias'] - - for key in config_f['autospec']: - config_opts[key] = config_f['autospec'].getboolean(key) - - # Rewrite the configuration file in case of formatting changes since a - # configuration file may exist without any comments (either due to an older - # version of autospec or if it was user-created) - rewrite_config_opts(path) - - # Don't use the ChangeLog files if the giturl is set - # ChangeLog is just extra noise when we can already see the gitlog - if "package" in config_f.sections() and config_f['package'].get('giturl'): - keys = [] - for k, v in transforms.items(): - if v == "ChangeLog": - keys.append(k) - for k in keys: - transforms.pop(k) - - -def rewrite_config_opts(path): - """Rewrite options.conf file when an option has changed (verify_required for example).""" - config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) - config_f['package'] = get_metadata_conf() - config_f['autospec'] = {} - - # Populate missing configuration options - # (in case of a user-created options.conf) - missing = set(config_options.keys()).difference(set(config_opts.keys())) - for option in missing: - config_opts[option] = False - - for fname, comment in sorted(config_options.items()): - config_f.set('autospec', '# {}'.format(comment)) - config_f['autospec'][fname] = 'true' if config_opts[fname] else 'false' - - write_config(config_f, path) - - -def read_file(path, track=True): - """Read full file at path. - - If the file does not exist (or is not expected to exist) - in the package git repo, specify 'track=False'. - """ - try: - with open(path, "r") as f: - if track: - config_files.add(os.path.basename(path)) - return f.readlines() - except EnvironmentError: - return [] - - -def read_conf_file(path, track=True): - """Read configuration file at path. - - If the config file does not exist (or is not expected to exist) - in the package git repo, specify 'track=False'. - """ - lines = read_file(path, track=track) - return [l.strip() for l in lines if not l.strip().startswith("#") and l.split()] - - -def read_script_file(path, track=True): - """Read RPM script snippet file at path. - - Returns verbatim, except for possibly the first line. - - If the config file does not exist (or is not expected to exist) - in the package git repo, specify 'track=False'. - """ - lines = read_file(path, track=track) - if len(lines) > 0 and (lines[0].startswith('#!') or lines[0].startswith('# -*- ')): - lines = lines[1:] - # Remove any trailing whitespace and newlines. The newlines are later - # restored by writer functions. - return [line.rstrip() for line in lines] - - def read_pattern_conf(filename, dest, list_format=False, path=None): """Read a fail-pattern configuration file. @@ -513,531 +72,888 @@ def read_pattern_conf(filename, dest, list_format=False, path=None): dest[pattern] = package.rstrip() -def setup_patterns(path=None): - """Read each pattern configuration file and assign to the appropriate variable.""" - global failed_commands - global ignored_commands - global maven_jars - global gems - global license_hashes - global license_translations - global license_blacklist - global qt_modules - global cmake_modules +class Config(object): + """Class to handle autospec configuration.""" - read_pattern_conf("ignored_commands", ignored_commands, list_format=True, path=path) - read_pattern_conf("failed_commands", failed_commands, path=path) - read_pattern_conf("maven_jars", maven_jars, path=path) - read_pattern_conf("gems", gems, path=path) - read_pattern_conf("license_hashes", license_hashes, path=path) - read_pattern_conf("license_translations", license_translations, path=path) - read_pattern_conf("license_blacklist", license_blacklist, list_format=True, path=path) - read_pattern_conf("qt_modules", qt_modules, path=path) - read_pattern_conf("cmake_modules", cmake_modules, path=path) + def __init__(self): + """Initialize Default configuration settings.""" + self.extra_configure = "" + self.extra_configure32 = "" + self.extra_configure64 = "" + self.extra_configure_avx2 = "" + self.extra_configure_avx512 = "" + self.config_files = set() + self.parallel_build = " %{?_smp_mflags} " + self.urlban = "" + self.extra_make = "" + self.extra32_make = "" + self.extra_make_install = "" + self.extra_make32_install = "" + self.extra_cmake = "" + self.extra_cmake_openmpi = "" + self.cmake_srcdir = "" + self.subdir = "" + self.install_macro = "%make_install" + self.disable_static = "--disable-static" + self.prep_prepend = [] + self.build_prepend = [] + self.build_append = [] + self.make_prepend = [] + self.install_prepend = [] + self.install_append = [] + self.service_restart = [] + self.patches = [] + self.verpatches = OrderedDict() + self.extra_sources = [] + self.autoreconf = False + self.custom_desc = "" + self.custom_summ = "" + self.set_gopath = True + self.license_fetch = None + self.license_show = None + self.git_uri = None + self.os_packages = set() + self.config_file = None + self.old_version = None + self.old_patches = list() + self.old_keyid = None + self.profile_payload = None + self.signature = None + self.yum_conf = None + self.failed_pattern_dir = None + self.alias = None + self.failed_commands = {} + self.ignored_commands = {} + self.maven_jars = {} + self.gems = {} + self.license_hashes = {} + self.license_translations = {} + self.license_blacklist = {} + self.qt_modules = {} + self.cmake_modules = {} + self.cves = [] + self.conf_args_openmpi = '--program-prefix= --exec-prefix=$MPI_ROOT \\\n' \ + '--libdir=$MPI_LIB --bindir=$MPI_BIN --sbindir=$MPI_BIN --includedir=$MPI_INCLUDE \\\n' \ + '--datarootdir=$MPI_ROOT/share --mandir=$MPI_MAN -exec-prefix=$MPI_ROOT --sysconfdir=$MPI_SYSCONFIG \\\n' \ + '--build=x86_64-generic-linux-gnu --host=x86_64-generic-linux-gnu --target=x86_64-clr-linux-gnu ' + # Keep track of the package versions + self.versions = OrderedDict() + # Only parse the versions file once, and save the result for later + self.parsed_versions = OrderedDict() + # defines which files to rename and copy to autospec directory, + # used in commitmessage.py + self.transforms = { + 'changes': 'ChangeLog', + 'changelog.txt': 'ChangeLog', + 'changelog': 'ChangeLog', + 'change.log': 'ChangeLog', + 'ChangeLog.md': 'ChangeLog', + 'changes.rst': 'ChangeLog', + 'changes.txt': 'ChangeLog', + 'news': 'NEWS', + 'meson_options.txt': 'meson_options.txt' + } + self.config_opts = {} + self.config_options = { + "broken_c++": "extend flags with '-std=gnu++98", + "use_lto": "configure build for lto", + "use_avx2": "configure build for avx2", + "use_avx512": "configure build for avx512", + "keepstatic": "do not remove static libraries", + "asneeded": "unset %build LD_AS_NEEDED variable", + "allow_test_failures": "allow package to build with test failures", + "skip_tests": "Do not run test suite", + "no_autostart": "do not require autostart subpackage", + "optimize_size": "optimize build for size over speed", + "funroll-loops": "optimize build for speed over size", + "fast-math": "pass -ffast-math to compiler", + "insecure_build": "set flags to smallest -02 flags possible", + "conservative_flags": "set conservative build flags", + "broken_parallel_build": "disable parallelization during build", + "pgo": "set profile for pgo", + "use_clang": "add clang flags", + "32bit": "build 32 bit libraries", + "nostrip": "disable stripping binaries", + "verify_required": "require package verification for build", + "security_sensitive": "set flags for security-sensitive builds", + "so_to_lib": "add .so files to the lib package instead of dev", + "dev_requires_extras": "dev package requires the extras to be installed", + "autoupdate": "this package is trusted enough to automatically update (used by other tools)", + "compat": "this package is a library compatibility package and only ships versioned library files", + "nodebug": "do not generate debuginfo for this package", + "openmpi": "configure build also for openmpi" + } + # simple_pattern_pkgconfig patterns + # contains patterns for parsing build.log for missing dependencies + self.pkgconfig_pats = [ + (r"which: no qmake", "Qt"), + (r"XInput2 extension not found", "xi"), + (r"checking for UDEV\.\.\. no", "udev"), + (r"checking for UDEV\.\.\. no", "libudev"), + (r"XMLLINT not set and xmllint not found in path", "libxml-2.0"), + (r"error\: xml2-config not found", "libxml-2.0"), + (r"error: must install xorg-macros", "xorg-macros") + ] + # simple_pattern patterns + # contains patterns for parsing build.log for missing dependencies + self.simple_pats = [ + (r'warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/.*"', "docbook-xml"), + (r"gobject-introspection dependency was not found, gir cannot be generated.", "gobject-introspection-dev"), + (r"gobject-introspection dependency was not found, gir cannot be generated.", "glibc-bin"), + (r"Cannot find development files for any supported version of libnl", "libnl-dev"), + (r"/", "cmake"), + (r"\-\- Boost libraries:", "boost-dev"), + (r"XInput2 extension not found", "inputproto"), + (r"^WARNING: could not find 'runtest'$", "dejagnu"), + (r"^WARNING: could not find 'runtest'$", "expect"), + (r"^WARNING: could not find 'runtest'$", "tcl"), + (r"VignetteBuilder package required for checking but installed:", "R-knitr"), + (r"You must have XML::Parser installed", "perl(XML::Parser)"), + (r"checking for Apache .* module support", "httpd-dev"), + (r"checking for.*in -ljpeg... no", "libjpeg-turbo-dev"), + (r"fatal error\: zlib\.h\: No such file or directory", "zlib-dev"), + (r"\* tclsh failed", "tcl"), + (r"\/usr\/include\/python3\.[0-9]+m\/pyconfig.h", "python3-dev"), + (r"checking \"location of ncurses\.h file\"", "ncurses-dev"), + (r"Can't exec \"aclocal\"", "automake"), + (r"Can't exec \"aclocal\"", "libtool"), + (r"configure: error: no suitable Python interpreter found", "python3-dev"), + (r"Checking for header Python.h", "python3-dev"), + (r"configure: error: No curses header-files found", "ncurses-dev"), + (r" \/usr\/include\/python3\.", "python3-dev"), + (r"to compile python extensions", "python3-dev"), + (r"testing autoconf... not found", "autoconf"), + (r"configure\: error\: could not find Python headers", "python3-dev"), + (r"checking for libxml libraries", "libxml2-dev"), + (r"checking for slang.h... no", "slang-dev"), + (r"configure: error: no suitable Python interpreter found", "python3"), + (r"configure: error: pcre-config for libpcre not found", "pcre"), + (r"checking for OpenSSL", "openssl-dev"), + (r"Package systemd was not found in the pkg-config search path.", "systemd-dev"), + (r"Unable to find the requested Boost libraries.", "boost-dev"), + (r"libproc not found. Please configure without procps", "procps-ng-dev"), + (r"configure: error: glib2", "glib-dev"), + (r"C library 'efivar' not found", "efivar-dev"), + (r"Has header \"efi.h\": NO", "gnu-efi-dev"), + (r"ERROR: Could not execute Vala compiler", "vala"), + (r".*: error: HAVE_INTROSPECTION does not appear in AM_CONDITIONAL", 'gobject-introspection-dev') + ] + # failed_pattern patterns + # contains patterns for parsing build.log for missing dependencies + self.failed_pats = [ + (r" ! ([a-zA-Z:]+) is not installed", 0, 'perl'), + (r" ([a-zA-Z]+\:\:[a-zA-Z]+) not installed", 1, None), + (r"(?:Could|Did) (?:NOT|not) find ([a-zA-Z0-9]+)", 0, None), + (r" ([a-zA-Z0-9\-]*\.m4) not found", 0, None), + (r" exec: ([a-zA-Z0-9\-]+): not found", 0, None), + (r"([a-zA-Z0-9\-\_\.]*)\: command not found", 1, None), + (r"([a-zA-Z\-]*) (?:validation )?tool not found or not executable", 0, None), + (r"([a-zA-Z\-]+) [0-9\.]+ is required to configure this module; " + r"please install it or upgrade your CPAN\/CPANPLUS shell.", 0, None), + (r"-- (.*) not found.", 1, None), + (r".* /usr/bin/([a-zA-Z0-9-_]*).*not found", 0, None), + (r".*\.go:.*cannot find package \"(.*)\" in any of:", 0, 'go'), + (r"/usr/bin/env\: (.*)\: No such file or directory", 0, None), + (r"/usr/bin/python.*\: No module named (.*)", 0, None), + (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:\/]+)", 0, 'ruby table'), + (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:]+) ", 0, 'ruby'), + (r"Add the installation prefix of \"(.*)\" to CMAKE_PREFIX_PATH", 0, None), + (r"By not providing \"([a-zA-Z0-9]+).cmake\" in CMAKE_MODULE_PATH this project", 0, None), + (r"C library '(.*)' not found", 0, None), + (r"CMake Error at cmake\/modules\/([a-zA-Z0-9]+).cmake", 0, None), + (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC " r"\(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'), + (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None), + (r"Checking for (.*?)\.\.\.no", 0, None), + (r"Checking for (.*?)\s*: not found", 0, None), + (r"Checking for (.*?)\s>=.*\s*: not found", 0, None), + (r"Could not find '([a-zA-Z0-9\-\_]*)' \([~<>=]+ ([0-9.]+).*\) among [0-9]+ total gem", 0, 'ruby'), + (r"Could not find gem '([a-zA-Z0-9\-\_]+) \([~<>=0-9\.\, ]+\) ruby'", 0, 'ruby'), + (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None), + (r"Download error on https://pypi.python.org/simple/([a-zA-Z0-9\-\._:]+)/", 0, 'pypi'), + (r"Downloading https?://.*\.python\.org/packages/.*/.?/([A-Za-z]*)/.*", 0, None), + (r"ERROR: Could not find a valid gem '([a-zA-Z0-9\-\_\:]*)' \([>=]+ ([0-9.]+).*\)", 0, 'ruby'), + (r"ERROR: dependencies ['‘]([a-zA-Z0-9\-\.]*)['’].* are not available for package ['‘].*['’]", 0, 'R'), + (r"ERROR: dependencies ['‘].*['’], ['‘]([a-zA-Z0-9\-\.]*)['’],.* are not available for package ['‘].*['’]", 0, 'R'), + (r"ERROR: dependencies.*['‘]([a-zA-Z0-9\-\.]*)['’] are not available for package ['‘].*['’]", 0, 'R'), + (r"ERROR: dependency ['‘]([a-zA-Z0-9\-\.]*)['’] is not available for package ['‘].*['’]", 0, 'R'), + (r"Error: Unable to find (.*)", 0, None), + (r"Error: package ['‘]([a-zA-Z0-9\-\.]*)['’] required by", 0, 'R'), + (r"Gem::LoadError: Could not find '([a-zA-Z0-9\-\_]*)'", 0, 'ruby'), + (r"ImportError:.* No module named '?([a-zA-Z0-9\-\._]+)'?", 0, 'pypi'), + (r"ImportError\: ([a-zA-Z]+) module missing", 0, None), + (r"ImportError\: (?:No module|cannot import) named? (.*)", 0, None), + (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:\/\_]+)", 0, 'ruby table'), + (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:]+)/.*", 0, 'ruby'), + (r"ModuleNotFoundError.*No module named (.*)", 0, None), + (r"Native dependency '(.*)' not found", 0, "pkgconfig"), + (r"No library found for -l([a-zA-Z\-])", 0, None), + (r"No (?:matching distribution|local packages or working download links) found for ([a-zA-Z0-9\-\.\_]+)", 0, 'pypi'), + (r"No package '([a-zA-Z0-9\-:]*)' found", 0, 'pkgconfig'), + (r"No rule to make target `(.*)',", 0, None), + (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'), + (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'), + (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'), + (r"Program (.*) found: NO", 0, None), + (r"Target '[a-zA-Z0-9\-]' can't be generated as '(.*)' could not be found", 0, None), + (r"Unable to `import (.*)`", 0, None), + (r"Unable to find '(.*)'", 0, None), + (r"Unknown packages? ['‘]([a-zA-Z0-9\-]*)['’].* in Rd xrefs", 0, 'R'), + (r"WARNING: [a-zA-Z\-\_]+ dependency on ([a-zA-Z0-9\-\_:]*) \([<>=~]+ ([0-9.]+).*\) .*", 0, 'ruby'), + (r"Warning: prerequisite ([a-zA-Z:]+) [0-9\.]+ not found.", 0, 'perl'), + (r"Warning\: no usable ([a-zA-Z0-9]+) found", 0, None), + (r"You need ([a-zA-Z0-9\-\_]*) to build this program.", 1, None), + (r"[Dd]ependency (.*) found: NO \(tried pkgconfig(?: and cmake)?\)", 0, 'pkgconfig'), + (r"[Dd]ependency (.*) found: NO", 0, None), + (r"[a-zA-Z0-9\-:]* is not installed: cannot load such file -- rdoc/([a-zA-Z0-9\-:]*)", 0, 'ruby'), + (r"\-\- Could NOT find ([a-zA-Z0-9]+)", 0, None), + (r"\/bin\/ld: cannot find (-l[a-zA-Z0-9\_]+)", 0, None), + (r"^.*By not providing \"Find(.*).cmake\" in CMAKE_MODULE_PATH this.*$", 0, None), + (r"^.*Could not find a package configuration file provided by \"(.*)\".*$", 0, None), + (r"^.*\"(.*)\" with any of the following names.*$", 0, None), + (r"[Cc]hecking for (.*) (?:support|development files|with pkg-config)?\.\.\. [Nn]o", 0, None), + (r"checking (.*?)\.\.\. no", 0, None), + (r"checking for (.*) in default path\.\.\. not found", 0, None), + (r"checking for (.*)... configure: error", 0, None), + (r"checking for (.*?)\.\.\. no", 0, None), + (r"checking for [a-zA-Z0-9\_\-]+ in (.*?)\.\.\. no", 0, None), + (r"checking for library containing (.*)... no", 0, None), + (r"checking for perl module ([a-zA-Z:]+) [0-9\.]+... no", 0, 'perl'), + (r"configure: error: (?:pkg-config missing|Unable to locate) (.*)", 0, None), + (r"configure: error: ([a-zA-Z0-9]+) (?:is required to build|not found)", 0, None), + (r"configure: error: Cannot find (.*)\. Make sure", 0, None), + (r"fatal error\: (.*)\: No such file or directory", 0, None), + (r"make: ([a-zA-Z0-9].+): Command not found", 0, None), + (r"meson\.build\:[\d]+\:[\d]+\: ERROR: C library \'(.*)\' not found", 0, None), + (r"there is no package called ['‘]([a-zA-Z0-9\-\.]*)['’]", 0, 'R'), + (r"unable to execute '([a-zA-Z\-]*)': No such file or directory", 0, None), + (r"warning: failed to load external entity " r"\"(/usr/share/sgml/docbook/xsl-stylesheets)/.*\"", 0, None), + (r"which\: no ([a-zA-Z\-]*) in \(", 0, None), + (r"you may need to install the ([a-zA-Z0-9_\-:\.]*) module", 0, 'perl'), + ] + def get_metadata_conf(self): + """Gather package metadata from the tarball module.""" + metadata = {} + metadata['name'] = tarball.name + if self.urlban: + metadata['url'] = re.sub(self.urlban, "localhost", tarball.url) + metadata['archives'] = re.sub(self.urlban, "localhost", " ".join(tarball.archives)) + else: + metadata['url'] = tarball.url + metadata['archives'] = " ".join(tarball.archives) -def parse_existing_spec(path, name): - """Determine the old version, old patch list, old keyid, and cves from old spec file.""" - global old_version - global old_patches - global old_keyid - global cves + metadata['giturl'] = tarball.giturl + metadata['domain'] = tarball.domain - spec = os.path.join(path, "{}.spec".format(name)) - if not os.path.exists(spec): - return + if self.alias: + metadata['alias'] = self.alias + else: + metadata['alias'] = "" + return metadata - found_old_version = False - found_old_patches = False - ver_regex = r"^Version *: *(.*) *$" - patch_regex = r"^Patch[0-9]* *: *(.*) *$" + def rewrite_config_opts(self, path): + """Rewrite options.conf file when an option has changed (verify_required for example).""" + config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) + config_f['package'] = self.get_metadata_conf() + config_f['autospec'] = {} - # If git history exists, read the Version and Patch* spec header fields - # from the latest commit to take priority over the working copy. - cmd = ["git", "-C", path, "grep", "-E", "-h", ver_regex, "HEAD", spec] - result = subprocess.run(cmd, capture_output=True) - if result.returncode == 0: - # The first matching line is from the spec header (hopefully) - line = result.stdout.decode().split("\n")[0] - m = re.search(ver_regex, line) - if m: - old_version = m.group(1) - found_old_version = True + # Populate missing configuration options + # (in case of a user-created options.conf) + missing = set(self.config_options.keys()).difference(set(self.config_opts.keys())) + for option in missing: + self.config_opts[option] = False - cmd = ["git", "-C", path, "grep", "-E", "-h", patch_regex, "HEAD", spec] - result = subprocess.run(cmd, capture_output=True) - if result.returncode == 0: - lines = result.stdout.decode().split("\n") - for line in lines: - m = re.search(patch_regex, line) - if m: - old_patches.append(m.group(1).lower()) - found_old_patches = True + for fname, comment in sorted(self.config_options.items()): + config_f.set('autospec', '# {}'.format(comment)) + config_f['autospec'][fname] = 'true' if self.config_opts[fname] else 'false' + write_config(config_f, path) - with open_auto(spec, "r") as inp: - for line in inp.readlines(): - line = line.strip().replace("\r", "").replace("\n", "") - if "Source0 file verified with key" in line: - keyidx = line.find('0x') + 2 - old_keyid = line[keyidx:].split()[0] if keyidx > 2 else old_keyid - # As a fallback, read the Version and Patch* header fields from the - # working copy of the spec, in case a git repo does not exist. - m = re.search(ver_regex, line) - if m and not found_old_version: - old_version = m.group(1) - found_old_version = True - m = re.search(patch_regex, line) - if m and not found_old_patches: - old_patches.append(m.group(1).lower()) + def create_conf(self, path): + """Create options.conf file and use deprecated configuration files or defaults to populate.""" + config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) - # Ignore nopatch - for patch in patches: - patch = patch.lower() - if patch not in old_patches and patch.endswith(".patch") and patch.startswith("cve-"): - cves.append(patch.upper().split(".PATCH")[0]) + # first the metadata + config_f['package'] = self.get_metadata_conf() - -def parse_config_versions(path): - """Parse the versions configuration file.""" - global versions - global parsed_versions - - # Only actually parse it the first time around - if not parsed_versions: - for line in read_conf_file(os.path.join(path, "versions")): - # Simply whitespace-separated fields - fields = line.split() - version = fields.pop(0) - if len(fields): - url = fields.pop(0) + # next the options + config_f['autospec'] = {} + for fname, comment in sorted(self.config_options.items()): + config_f.set('autospec', '# {}'.format(comment)) + if os.path.exists(fname): + config_f['autospec'][fname] = 'true' + os.remove(fname) else: - url = "" - # Catch and report duplicate URLs in the versions file. Don't stop, - # but assume only the first one is valid and drop the rest. - if version in parsed_versions and url != parsed_versions[version]: - print_warning("Already have a URL defined for {}: {}" - .format(version, parsed_versions[version])) - print_warning("Dropping {}, but you should check my work" - .format(url)) - else: - parsed_versions[version] = url - if len(fields): - print_warning("Extra fields detected in `versions` file entry:\n{}" - .format(line)) - print_warning("I'll delete them, but you should check my work") + config_f['autospec'][fname] = 'false' - # We'll combine what we just parsed from the versions file with any other - # versions that have already been defined, most likely the version actually - # provided in the Makefile's URL variable, so we don't drop any. - for version in parsed_versions: - versions[version] = parsed_versions[version] + # default lto to true for new things + config_f['autospec']['use_lto'] = 'true' - return versions + # renamed options need special care + if os.path.exists("skip_test_suite"): + config_f['autospec']['skip_tests'] = 'true' + os.remove("skip_test_suite") + write_config(config_f, path) + def create_buildreq_cache(self, path, version): + """Make the buildreq_cache file.""" + content = self.read_conf_file(os.path.join(path, "buildreq_cache")) + # don't create an empty cache file + if len(buildreq.buildreqs_cache) < 1: + try: + # file was possibly added to git so we should clean it up + os.unlink(content) + except Exception: + pass + return + if not content: + pkgs = sorted(buildreq.buildreqs_cache) + else: + pkgs = sorted(set(content[1:]).union(buildreq.buildreqs_cache)) + with open(os.path.join(path, 'buildreq_cache'), "w") as cachefile: + cachefile.write("\n".join([version] + pkgs)) + self.config_files.add('buildreq_cache') -def parse_config_files(path, bump, filemanager, version): - """Parse the various configuration files that may exist in the package directory.""" - global extra_configure - global extra_configure32 - global extra_configure64 - global extra_configure_avx2 - global extra_configure_avx512 - global extra_configure_openmpi - global config_files - global extra_sources - global parallel_build - global license_fetch - global license_show - global git_uri - global os_packages - global urlban - global config_file - global profile_payload - global config_opts - global extra_make - global extra32_make - global extra_make_install - global extra_make32_install - global extra_cmake - global extra_cmake_openmpi - global cmake_srcdir - global subdir - global install_macro - global disable_static - global prep_prepend - global build_prepend - global build_append - global make_prepend - global install_prepend - global install_append - global service_restart - global patches - global autoreconf - global set_gopath - global yum_conf - global custom_desc - global custom_summ - global failed_pattern_dir - global versions + def create_versions(self, path, versions): + """Make versions file.""" + with open(os.path.join(path, "versions"), 'w') as vfile: + for version in versions: + vfile.write(version) + if versions[version]: + vfile.write('\t' + versions[version]) + vfile.write('\n') + self.config_files.add("versions") - packages_file = None + def read_config_opts(self, path): + """Read config options from path/options.conf.""" + opts_path = os.path.join(path, 'options.conf') + if not os.path.exists(opts_path): + self.create_conf(path) - # Require autospec.conf for additional features - if os.path.exists(config_file): - config = configparser.ConfigParser(interpolation=None) - config.read(config_file) - - if "autospec" not in config.sections(): - print("Missing autospec section..") + config_f = configparser.ConfigParser(interpolation=None) + config_f.read(opts_path) + if "autospec" not in config_f.sections(): + print("Missing autospec section in options.conf") sys.exit(1) - git_uri = config['autospec'].get('git', None) - license_fetch = config['autospec'].get('license_fetch', None) - license_show = config['autospec'].get('license_show', None) - packages_file = config['autospec'].get('packages_file', None) - yum_conf = config['autospec'].get('yum_conf', None) - failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None) + if 'package' in config_f.sections() and config_f['package'].get('alias'): + self.alias = config_f['package']['alias'] - # support reading the local files relative to config_file - if packages_file and not os.path.isabs(packages_file): - packages_file = os.path.join(os.path.dirname(config_file), packages_file) - if yum_conf and not os.path.isabs(yum_conf): - yum_conf = os.path.join(os.path.dirname(config_file), yum_conf) - if failed_pattern_dir and not os.path.isabs(failed_pattern_dir): - failed_pattern_dir = os.path.join(os.path.dirname(config_file), failed_pattern_dir) + for key in config_f['autospec']: + self.config_opts[key] = config_f['autospec'].getboolean(key) - if not packages_file: - print("Warning: Set [autospec][packages_file] path to package list file for " - "requires validation") - packages_file = os.path.join(os.path.dirname(config_file), "packages") + # Rewrite the configuration file in case of formatting changes since a + # configuration file may exist without any comments (either due to an older + # version of autospec or if it was user-created) + self.rewrite_config_opts(path) - urlban = config['autospec'].get('urlban', None) + # Don't use the ChangeLog files if the giturl is set + # ChangeLog is just extra noise when we can already see the gitlog + if "package" in config_f.sections() and config_f['package'].get('giturl'): + keys = [] + for k, v in self.transforms.items(): + if v == "ChangeLog": + keys.append(k) + for k in keys: + self.transforms.pop(k) - # Read values from options.conf (and deprecated files) and rewrite as necessary - read_config_opts(path) + def read_file(self, path, track=True): + """Read full file at path. - if not git_uri: - print("Warning: Set [autospec][git] upstream template for remote git URI configuration") - if not license_fetch: - print("Warning: Set [autospec][license_fetch] uri for license fetch support") - if not license_show: - print("Warning: Set [autospec][license_show] uri for license link check support") - if not yum_conf: - print("Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation") - yum_conf = os.path.join(os.path.dirname(config_file), "image-creator/yum.conf") + If the file does not exist (or is not expected to exist) + in the package git repo, specify 'track=False'. + """ + try: + with open(path, "r") as f: + if track: + self.config_files.add(os.path.basename(path)) + return f.readlines() + except EnvironmentError: + return [] - if packages_file: - os_packages = set(read_conf_file(packages_file, track=False)) - else: - os_packages = set(read_conf_file("~/packages", track=False)) + def read_conf_file(self, path, track=True): + """Read configuration file at path. - wrapper = textwrap.TextWrapper() - wrapper.initial_indent = "# " - wrapper.subsequent_indent = "# " + If the config file does not exist (or is not expected to exist) + in the package git repo, specify 'track=False'. + """ + lines = self.read_file(path, track=track) + return [l.strip() for l in lines if not l.strip().startswith("#") and l.split()] - def write_default_conf_file(name, description): + def read_script_file(self, path, track=True): + """Read RPM script snippet file at path. + + Returns verbatim, except for possibly the first line. + + If the config file does not exist (or is not expected to exist) + in the package git repo, specify 'track=False'. + """ + lines = self.read_file(path, track=track) + if len(lines) > 0 and (lines[0].startswith('#!') or lines[0].startswith('# -*- ')): + lines = lines[1:] + # Remove any trailing whitespace and newlines. The newlines are later + # restored by writer functions. + return [line.rstrip() for line in lines] + + def setup_patterns(self, path=None): + """Read each pattern configuration file and assign to the appropriate variable.""" + read_pattern_conf("ignored_commands", self.ignored_commands, list_format=True, path=path) + read_pattern_conf("failed_commands", self.failed_commands, path=path) + read_pattern_conf("maven_jars", self.maven_jars, path=path) + read_pattern_conf("gems", self.gems, path=path) + read_pattern_conf("license_hashes", self.license_hashes, path=path) + read_pattern_conf("license_translations", self.license_translations, path=path) + read_pattern_conf("license_blacklist", self.license_blacklist, list_format=True, path=path) + read_pattern_conf("qt_modules", self.qt_modules, path=path) + read_pattern_conf("cmake_modules", self.cmake_modules, path=path) + + def parse_existing_spec(self, path, name): + """Determine the old version, old patch list, old keyid, and cves from old spec file.""" + spec = os.path.join(path, "{}.spec".format(name)) + if not os.path.exists(spec): + return + + found_old_version = False + found_old_patches = False + ver_regex = r"^Version *: *(.*) *$" + patch_regex = r"^Patch[0-9]* *: *(.*) *$" + + # If git history exists, read the Version and Patch* spec header fields + # from the latest commit to take priority over the working copy. + cmd = ["git", "-C", path, "grep", "-E", "-h", ver_regex, "HEAD", spec] + result = subprocess.run(cmd, capture_output=True) + if result.returncode == 0: + # The first matching line is from the spec header (hopefully) + line = result.stdout.decode().split("\n")[0] + m = re.search(ver_regex, line) + if m: + self.old_version = m.group(1) + found_old_version = True + + cmd = ["git", "-C", path, "grep", "-E", "-h", patch_regex, "HEAD", spec] + result = subprocess.run(cmd, capture_output=True) + if result.returncode == 0: + lines = result.stdout.decode().split("\n") + for line in lines: + m = re.search(patch_regex, line) + if m: + self.old_patches.append(m.group(1).lower()) + found_old_patches = True + + with open_auto(spec, "r") as inp: + for line in inp.readlines(): + line = line.strip().replace("\r", "").replace("\n", "") + if "Source0 file verified with key" in line: + keyidx = line.find('0x') + 2 + self.old_keyid = line[keyidx:].split()[0] if keyidx > 2 else self.old_keyid + # As a fallback, read the Version and Patch* header fields from the + # working copy of the spec, in case a git repo does not exist. + m = re.search(ver_regex, line) + if m and not found_old_version: + self.old_version = m.group(1) + found_old_version = True + m = re.search(patch_regex, line) + if m and not found_old_patches: + self.old_patches.append(m.group(1).lower()) + + # Ignore nopatch + for patch in self.patches: + patch = patch.lower() + if patch not in self.old_patches and patch.endswith(".patch") and patch.startswith("cve-"): + self.cves.append(patch.upper().split(".PATCH")[0]) + + def parse_config_versions(self, path): + """Parse the versions configuration file.""" + # Only actually parse it the first time around + if not self.parsed_versions: + for line in self.read_conf_file(os.path.join(path, "versions")): + # Simply whitespace-separated fields + fields = line.split() + version = fields.pop(0) + if len(fields): + url = fields.pop(0) + else: + url = "" + # Catch and report duplicate URLs in the versions file. Don't stop, + # but assume only the first one is valid and drop the rest. + if version in self.parsed_versions and url != self.parsed_versions[version]: + print_warning("Already have a URL defined for {}: {}" + .format(version, self.parsed_versions[version])) + print_warning("Dropping {}, but you should check my work" + .format(url)) + else: + self.parsed_versions[version] = url + if len(fields): + print_warning("Extra fields detected in `versions` file entry:\n{}" + .format(line)) + print_warning("I'll delete them, but you should check my work") + + # We'll combine what we just parsed from the versions file with any other + # versions that have already been defined, most likely the version actually + # provided in the Makefile's URL variable, so we don't drop any. + for version in self.parsed_versions: + self.versions[version] = self.parsed_versions[version] + + return self.versions + + def write_default_conf_file(self, path, name, wrapper, description): """Write default configuration file with description to file name.""" - config_files.add(name) + self.config_files.add(name) filename = os.path.join(path, name) if os.path.isfile(filename): return write_out(filename, wrapper.fill(description) + "\n") - write_default_conf_file("buildreq_ban", - "This file contains build requirements that get picked up but are " - "undesirable. One entry per line, no whitespace.") - write_default_conf_file("pkgconfig_ban", - "This file contains pkgconfig build requirements that get picked up but" - " are undesirable. One entry per line, no whitespace.") - write_default_conf_file("requires_ban", - "This file contains runtime requirements that get picked up but are " - "undesirable. One entry per line, no whitespace.") - write_default_conf_file("buildreq_add", - "This file contains additional build requirements that did not get " - "picked up automatically. One name per line, no whitespace.") - write_default_conf_file("pkgconfig_add", - "This file contains additional pkgconfig build requirements that did " - "not get picked up automatically. One name per line, no whitespace.") - write_default_conf_file("requires_add", - "This file contains additional runtime requirements that did not get " - "picked up automatically. One name per line, no whitespace.") - write_default_conf_file("excludes", - "This file contains the output files that need %exclude. Full path " - "names, one per line.") + def parse_config_files(self, path, bump, filemanager, version): + """Parse the various configuration files that may exist in the package directory.""" + packages_file = None - content = read_conf_file(os.path.join(path, "release")) - if content and content[0]: - r = int(content[0]) - if bump: - r += 1 - tarball.release = str(r) - print("Release :", tarball.release) + # Require autospec.conf for additional features + if os.path.exists(self.config_file): + config = configparser.ConfigParser(interpolation=None) + config.read(self.config_file) - content = read_conf_file(os.path.join(path, "extra_sources")) - for source in content: - fields = source.split(maxsplit=1) - print("Adding additional source file: %s" % fields[0]) - config_files.add(os.path.basename(fields[0])) - extra_sources.append(fields) + if "autospec" not in config.sections(): + print("Missing autospec section..") + sys.exit(1) - content = read_conf_file(os.path.join(path, "buildreq_ban")) - for banned in content: - print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + self.git_uri = config['autospec'].get('git', None) + self.license_fetch = config['autospec'].get('license_fetch', None) + self.license_show = config['autospec'].get('license_show', None) + packages_file = config['autospec'].get('packages_file', None) + self.yum_conf = config['autospec'].get('yum_conf', None) + self.failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None) - content = read_conf_file(os.path.join(path, "pkgconfig_ban")) - for banned in content: - banned = "pkgconfig(%s)" % banned - print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + # support reading the local files relative to config_file + if packages_file and not os.path.isabs(packages_file): + packages_file = os.path.join(os.path.dirname(self.config_file), packages_file) + if self.yum_conf and not os.path.isabs(self.yum_conf): + self.yum_conf = os.path.join(os.path.dirname(self.config_file), self.yum_conf) + if self.failed_pattern_dir and not os.path.isabs(self.failed_pattern_dir): + self.failed_pattern_dir = os.path.join(os.path.dirname(self.config_file), self.failed_pattern_dir) - content = read_conf_file(os.path.join(path, "requires_ban")) - for banned in content: - print("Banning runtime requirement: %s." % banned) - buildreq.banned_requires.add(banned) - buildreq.requires.discard(banned) + if not packages_file: + print("Warning: Set [autospec][packages_file] path to package list file for " + "requires validation") + packages_file = os.path.join(os.path.dirname(self.config_file), "packages") - content = read_conf_file(os.path.join(path, "buildreq_add")) - for extra in content: - print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + self.urlban = config['autospec'].get('urlban', None) - cache_file = os.path.join(path, "buildreq_cache") - content = read_conf_file(cache_file) - if content and content[0] == version: - for extra in content[1:]: - print("Adding additional build (cache) requirement: %s." % extra) + # Read values from options.conf (and deprecated files) and rewrite as necessary + self.read_config_opts(path) + + if not self.git_uri: + print("Warning: Set [autospec][git] upstream template for remote git URI configuration") + if not self.license_fetch: + print("Warning: Set [autospec][license_fetch] uri for license fetch support") + if not self.license_show: + print("Warning: Set [autospec][license_show] uri for license link check support") + if not self.yum_conf: + print("Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation") + self.yum_conf = os.path.join(os.path.dirname(self.config_file), "image-creator/yum.conf") + + if packages_file: + self.os_packages = set(self.read_conf_file(packages_file, track=False)) + else: + self.os_packages = set(self.read_conf_file("~/packages", track=False)) + + wrapper = textwrap.TextWrapper() + wrapper.initial_indent = "# " + wrapper.subsequent_indent = "# " + + self.write_default_conf_file(path, "buildreq_ban", wrapper, + "This file contains build requirements that get picked up but are " + "undesirable. One entry per line, no whitespace.") + self.write_default_conf_file(path, "pkgconfig_ban", wrapper, + "This file contains pkgconfig build requirements that get picked up but" + " are undesirable. One entry per line, no whitespace.") + self.write_default_conf_file(path, "requires_ban", wrapper, + "This file contains runtime requirements that get picked up but are " + "undesirable. One entry per line, no whitespace.") + self.write_default_conf_file(path, "buildreq_add", wrapper, + "This file contains additional build requirements that did not get " + "picked up automatically. One name per line, no whitespace.") + self.write_default_conf_file(path, "pkgconfig_add", wrapper, + "This file contains additional pkgconfig build requirements that did " + "not get picked up automatically. One name per line, no whitespace.") + self.write_default_conf_file(path, "requires_add", wrapper, + "This file contains additional runtime requirements that did not get " + "picked up automatically. One name per line, no whitespace.") + self.write_default_conf_file(path, "excludes", wrapper, + "This file contains the output files that need %exclude. Full path " + "names, one per line.") + + content = self.read_conf_file(os.path.join(path, "release")) + if content and content[0]: + r = int(content[0]) + if bump: + r += 1 + tarball.release = str(r) + print("Release :", tarball.release) + + content = self.read_conf_file(os.path.join(path, "extra_sources")) + for source in content: + fields = source.split(maxsplit=1) + print("Adding additional source file: %s" % fields[0]) + self.config_files.add(os.path.basename(fields[0])) + self.extra_sources.append(fields) + + content = self.read_conf_file(os.path.join(path, "buildreq_ban")) + for banned in content: + print("Banning build requirement: %s." % banned) + buildreq.banned_buildreqs.add(banned) + buildreq.buildreqs.discard(banned) + buildreq.buildreqs_cache.discard(banned) + + content = self.read_conf_file(os.path.join(path, "pkgconfig_ban")) + for banned in content: + banned = "pkgconfig(%s)" % banned + print("Banning build requirement: %s." % banned) + buildreq.banned_buildreqs.add(banned) + buildreq.buildreqs.discard(banned) + buildreq.buildreqs_cache.discard(banned) + + content = self.read_conf_file(os.path.join(path, "requires_ban")) + for banned in content: + print("Banning runtime requirement: %s." % banned) + buildreq.banned_requires.add(banned) + buildreq.requires.discard(banned) + + content = self.read_conf_file(os.path.join(path, "buildreq_add")) + for extra in content: + print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) - else: - try: - os.unlink(cache_file) - except FileNotFoundError: - pass - except Exception as e: - print_warning(f"Unable to remove buildreq_cache file: {e}") - content = read_conf_file(os.path.join(path, "pkgconfig_add")) - for extra in content: - extra = "pkgconfig(%s)" % extra - print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + cache_file = os.path.join(path, "buildreq_cache") + content = self.read_conf_file(cache_file) + if content and content[0] == version: + for extra in content[1:]: + print("Adding additional build (cache) requirement: %s." % extra) + buildreq.add_buildreq(extra) + else: + try: + os.unlink(cache_file) + except FileNotFoundError: + pass + except Exception as e: + print_warning(f"Unable to remove buildreq_cache file: {e}") - content = read_conf_file(os.path.join(path, "requires_add")) - for extra in content: - print("Adding additional runtime requirement: %s." % extra) - buildreq.add_requires(extra, override=True) + content = self.read_conf_file(os.path.join(path, "pkgconfig_add")) + for extra in content: + extra = "pkgconfig(%s)" % extra + print("Adding additional build requirement: %s." % extra) + buildreq.add_buildreq(extra) - content = read_conf_file(os.path.join(path, "excludes")) - for exclude in content: - print("%%exclude for: %s." % exclude) - filemanager.excludes += content + content = self.read_conf_file(os.path.join(path, "requires_add")) + for extra in content: + print("Adding additional runtime requirement: %s." % extra) + buildreq.add_requires(extra, self.os_packages, override=True) - content = read_conf_file(os.path.join(path, "extras")) - for extra in content: - print("extras for : %s." % extra) - filemanager.extras += content + content = self.read_conf_file(os.path.join(path, "excludes")) + for exclude in content: + print("%%exclude for: %s." % exclude) + filemanager.excludes += content - for fname in os.listdir(path): - if not re.search('.+_extras$', fname) or fname == "dev_extras": - continue - content = {} - content['files'] = read_conf_file(os.path.join(path, fname)) - if not content: - print_warning(f"Error reading custom extras file: {fname}") - continue - req_file = os.path.join(path, f'{fname}_requires') - if os.path.isfile(req_file): - content['requires'] = read_conf_file(req_file) - name = fname[:-len("_extras")] - print(f"extras-{name} for {content['files']}") - filemanager.custom_extras["extras-" + f"{name}"] = content + content = self.read_conf_file(os.path.join(path, "extras")) + for extra in content: + print("extras for : %s." % extra) + filemanager.extras += content - content = read_conf_file(os.path.join(path, "dev_extras")) - for extra in content: - print("dev for : %s." % extra) - filemanager.dev_extras += content + for fname in os.listdir(path): + if not re.search('.+_extras$', fname) or fname == "dev_extras": + continue + content = {} + content['files'] = self.read_conf_file(os.path.join(path, fname)) + if not content: + print_warning(f"Error reading custom extras file: {fname}") + continue + req_file = os.path.join(path, f'{fname}_requires') + if os.path.isfile(req_file): + content['requires'] = self.read_conf_file(req_file) + name = fname[:-len("_extras")] + print(f"extras-{name} for {content['files']}") + filemanager.custom_extras["extras-" + f"{name}"] = content - content = read_conf_file(os.path.join(path, "setuid")) - for suid in content: - print("setuid for : %s." % suid) - filemanager.setuid += content + content = self.read_conf_file(os.path.join(path, "dev_extras")) + for extra in content: + print("dev for : %s." % extra) + filemanager.dev_extras += content - content = read_conf_file(os.path.join(path, "attrs")) - for line in content: - attr = line.split() - filename = attr.pop() - print("%attr({0},{1},{2}) for: {3}".format( - attr[0], attr[1], attr[2], filename)) - filemanager.attrs[filename] = attr + content = self.read_conf_file(os.path.join(path, "setuid")) + for suid in content: + print("setuid for : %s." % suid) + filemanager.setuid += content - patches += read_conf_file(os.path.join(path, "series")) - pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches] - cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(aclocal.m4)|(configure.ac|configure.in))\" %s" % " ".join(pfiles) # noqa: W605 - if patches and call(cmd, - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0: - autoreconf = True + content = self.read_conf_file(os.path.join(path, "attrs")) + for line in content: + attr = line.split() + filename = attr.pop() + print("%attr({0},{1},{2}) for: {3}".format( + attr[0], attr[1], attr[2], filename)) + filemanager.attrs[filename] = attr - # Parse the version-specific patch lists - update_security_sensitive = False - for version in versions: - verpatches[version] = read_conf_file(os.path.join(path, '.'.join(['series', version]))) - if any(p.lower().startswith('cve-') for p in verpatches[version]): + self.patches += self.read_conf_file(os.path.join(path, "series")) + pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in self.patches] + cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(aclocal.m4)|(configure.ac|configure.in))\" %s" % " ".join(pfiles) # noqa: W605 + if self.patches and call(cmd, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0: + self.autoreconf = True + + # Parse the version-specific patch lists + update_security_sensitive = False + for version in self.versions: + self.verpatches[version] = self.read_conf_file(os.path.join(path, '.'.join(['series', version]))) + if any(p.lower().startswith('cve-') for p in self.verpatches[version]): + update_security_sensitive = True + + if any(p.lower().startswith('cve-') for p in self.patches): update_security_sensitive = True - if any(p.lower().startswith('cve-') for p in patches): - update_security_sensitive = True + if update_security_sensitive: + self.config_opts['security_sensitive'] = True + self.rewrite_config_opts(path) - if update_security_sensitive: - config_opts['security_sensitive'] = True - rewrite_config_opts(path) + content = self.read_conf_file(os.path.join(path, "configure")) + self.extra_configure = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure")) - extra_configure = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure32")) + self.extra_configure32 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure32")) - extra_configure32 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure64")) + self.extra_configure64 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure64")) - extra_configure64 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure_avx2")) + self.extra_configure_avx2 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure_avx2")) - extra_configure_avx2 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure_avx512")) + self.extra_configure_avx512 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure_avx512")) - extra_configure_avx512 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure_openmpi")) + self.extra_configure_openmpi = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure_openmpi")) - extra_configure_openmpi = " \\\n".join(content) + if self.config_opts["keepstatic"]: + self.disable_static = "" + if self.config_opts['broken_parallel_build']: + self.parallel_build = "" - if config_opts["keepstatic"]: - disable_static = "" - if config_opts['broken_parallel_build']: - parallel_build = "" + content = self.read_conf_file(os.path.join(path, "make_args")) + if content: + self.extra_make = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make_args")) - if content: - extra_make = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "make32_args")) + if content: + self.extra32_make = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make32_args")) - if content: - extra32_make = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "make_install_args")) + if content: + self.extra_make_install = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make_install_args")) - if content: - extra_make_install = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "make32_install_args")) + if content: + self.extra_make32_install = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make32_install_args")) - if content: - extra_make32_install = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "install_macro")) + if content and content[0]: + self.install_macro = content[0] - content = read_conf_file(os.path.join(path, "install_macro")) - if content and content[0]: - install_macro = content[0] + content = self.read_conf_file(os.path.join(path, "cmake_args")) + if content: + self.extra_cmake = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "cmake_args")) - if content: - extra_cmake = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "cmake_args_openmpi")) + if content: + self.extra_cmake_openmpi = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "cmake_args_openmpi")) - if content: - extra_cmake_openmpi = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "cmake_srcdir")) + if content and content[0]: + self.cmake_srcdir = content[0] - content = read_conf_file(os.path.join(path, "cmake_srcdir")) - if content and content[0]: - cmake_srcdir = content[0] + content = self.read_conf_file(os.path.join(path, "subdir")) + if content and content[0]: + self.subdir = content[0] - content = read_conf_file(os.path.join(path, "subdir")) - if content and content[0]: - subdir = content[0] + content = self.read_conf_file(os.path.join(path, "build_pattern")) + if content and content[0]: + buildpattern.set_build_pattern(content[0], 20) + self.autoreconf = False - content = read_conf_file(os.path.join(path, "build_pattern")) - if content and content[0]: - buildpattern.set_build_pattern(content[0], 20) - autoreconf = False + content = self.read_script_file(os.path.join(path, "make_check_command")) + if content: + check.tests_config = '\n'.join(content) - content = read_script_file(os.path.join(path, "make_check_command")) - if content: - check.tests_config = '\n'.join(content) + content = self.read_conf_file(os.path.join(path, tarball.name + ".license")) + if content and content[0]: + words = content[0].split() + for word in words: + if word.find(":") < 0: + if not license.add_license(word, self.license_translations, self.license_blacklist): + print_warning("{}: blacklisted license {} ignored.".format(tarball.name + ".license", word)) - content = read_conf_file(os.path.join(path, tarball.name + ".license")) - if content and content[0]: - words = content[0].split() - for word in words: - if word.find(":") < 0: - if not license.add_license(word): - print_warning("{}: blacklisted license {} ignored.".format(tarball.name + ".license", word)) + content = self.read_conf_file(os.path.join(path, "golang_libpath")) + if content and content[0]: + tarball.golibpath = content[0] + print("golibpath : {}".format(tarball.golibpath)) - content = read_conf_file(os.path.join(path, "golang_libpath")) - if content and content[0]: - tarball.golibpath = content[0] - print("golibpath : {}".format(tarball.golibpath)) + if self.config_opts['use_clang']: + self.config_opts['funroll-loops'] = False + buildreq.add_buildreq("llvm") - if config_opts['use_clang']: - config_opts['funroll-loops'] = False - buildreq.add_buildreq("llvm") + if self.config_opts['32bit']: + buildreq.add_buildreq("glibc-libc32") + buildreq.add_buildreq("glibc-dev32") + buildreq.add_buildreq("gcc-dev32") + buildreq.add_buildreq("gcc-libgcc32") + buildreq.add_buildreq("gcc-libstdc++32") - if config_opts['32bit']: - buildreq.add_buildreq("glibc-libc32") - buildreq.add_buildreq("glibc-dev32") - buildreq.add_buildreq("gcc-dev32") - buildreq.add_buildreq("gcc-libgcc32") - buildreq.add_buildreq("gcc-libstdc++32") + if self.config_opts['openmpi']: + buildreq.add_buildreq("openmpi-dev") + buildreq.add_buildreq("modules") + # MPI testsuites generally require "openssh" + buildreq.add_buildreq("openssh") - if config_opts['openmpi']: - buildreq.add_buildreq("openmpi-dev") - buildreq.add_buildreq("modules") - # MPI testsuites generally require "openssh" - buildreq.add_buildreq("openssh") + self.prep_prepend = self.read_script_file(os.path.join(path, "prep_prepend")) + if os.path.isfile(os.path.join(path, "prep_append")): + os.rename(os.path.join(path, "prep_append"), os.path.join(path, "build_prepend")) + self.make_prepend = self.read_script_file(os.path.join(path, "make_prepend")) + self.build_prepend = self.read_script_file(os.path.join(path, "build_prepend")) + self.build_append = self.read_script_file(os.path.join(path, "build_append")) + self.install_prepend = self.read_script_file(os.path.join(path, "install_prepend")) + if os.path.isfile(os.path.join(path, "make_install_append")): + os.rename(os.path.join(path, "make_install_append"), os.path.join(path, "install_append")) + self.install_append = self.read_script_file(os.path.join(path, "install_append")) + self.service_restart = self.read_conf_file(os.path.join(path, "service_restart")) - prep_prepend = read_script_file(os.path.join(path, "prep_prepend")) - if os.path.isfile(os.path.join(path, "prep_append")): - os.rename(os.path.join(path, "prep_append"), os.path.join(path, "build_prepend")) - make_prepend = read_script_file(os.path.join(path, "make_prepend")) - build_prepend = read_script_file(os.path.join(path, "build_prepend")) - build_append = read_script_file(os.path.join(path, "build_append")) - install_prepend = read_script_file(os.path.join(path, "install_prepend")) - if os.path.isfile(os.path.join(path, "make_install_append")): - os.rename(os.path.join(path, "make_install_append"), os.path.join(path, "install_append")) - install_append = read_script_file(os.path.join(path, "install_append")) - service_restart = read_conf_file(os.path.join(path, "service_restart")) + self.profile_payload = self.read_script_file(os.path.join(path, "profile_payload")) - profile_payload = read_script_file(os.path.join(path, "profile_payload")) + self.custom_desc = self.read_conf_file(os.path.join(path, "description")) + self.custom_summ = self.read_conf_file(os.path.join(path, "summary")) - custom_desc = read_conf_file(os.path.join(path, "description")) - custom_summ = read_conf_file(os.path.join(path, "summary")) - - -def load_specfile(specfile): - """Load specfile object with configuration.""" - specfile.urlban = urlban - specfile.keepstatic = config_opts['keepstatic'] - specfile.no_autostart = config_opts['no_autostart'] - specfile.extra_make = extra_make - specfile.extra32_make = extra32_make - specfile.extra_make_install = extra_make_install - specfile.extra_make32_install = extra_make32_install - specfile.extra_cmake = extra_cmake - specfile.extra_cmake_openmpi = extra_cmake_openmpi - specfile.cmake_srcdir = cmake_srcdir or specfile.cmake_srcdir - specfile.subdir = subdir - specfile.install_macro = install_macro - specfile.disable_static = disable_static - specfile.prep_prepend = prep_prepend - specfile.build_prepend = build_prepend - specfile.build_append = build_append - specfile.make_prepend = make_prepend - specfile.install_prepend = install_prepend - specfile.install_append = install_append - specfile.service_restart = service_restart - specfile.extra_sources = extra_sources - specfile.patches = patches - specfile.verpatches = verpatches - specfile.autoreconf = autoreconf - specfile.set_gopath = set_gopath + def load_specfile(self, specfile): + """Load specfile object with configuration.""" + specfile.urlban = self.urlban + specfile.keepstatic = self.config_opts['keepstatic'] + specfile.no_autostart = self.config_opts['no_autostart'] + specfile.extra_make = self.extra_make + specfile.extra32_make = self.extra32_make + specfile.extra_make_install = self.extra_make_install + specfile.extra_make32_install = self.extra_make32_install + specfile.extra_cmake = self.extra_cmake + specfile.extra_cmake_openmpi = self.extra_cmake_openmpi + specfile.cmake_srcdir = self.cmake_srcdir or specfile.cmake_srcdir + specfile.subdir = self.subdir + specfile.install_macro = self.install_macro + specfile.disable_static = self.disable_static + specfile.prep_prepend = self.prep_prepend + specfile.build_prepend = self.build_prepend + specfile.build_append = self.build_append + specfile.make_prepend = self.make_prepend + specfile.install_prepend = self.install_prepend + specfile.install_append = self.install_append + specfile.service_restart = self.service_restart + specfile.extra_sources = self.extra_sources + specfile.patches = self.patches + specfile.verpatches = self.verpatches + specfile.autoreconf = self.autoreconf + specfile.set_gopath = self.set_gopath diff --git a/autospec/files.py b/autospec/files.py index 437ef8f..b6add8f 100644 --- a/autospec/files.py +++ b/autospec/files.py @@ -24,7 +24,6 @@ import re from collections import OrderedDict import build -import config import tarball import util @@ -32,8 +31,9 @@ import util class FileManager(object): """Class to handle spec file %files section management.""" - def __init__(self): + def __init__(self, config): """Set defaults for FileManager.""" + self.config = config self.packages = OrderedDict() # per sub-package file list for spec purposes self.files = set() # global file set to weed out dupes self.files_blacklist = set() @@ -89,7 +89,7 @@ class FileManager(object): def compat_exclude(self, filename): """Exclude non-library files if the package is for compatability.""" - if not config.config_opts.get("compat"): + if not self.config.config_opts.get("compat"): return False patterns = [ @@ -228,8 +228,8 @@ class FileManager(object): # if configured to do so, add .so files to the lib package instead of # the dev package. THis is useful for packages with a plugin # architecture like elfutils and mesa. - so_dest = 'lib' if config.config_opts.get('so_to_lib') else 'dev' - so_dest_ompi = 'openmpi' if config.config_opts.get('so_to_lib') else 'dev' + so_dest = 'lib' if self.config.config_opts.get('so_to_lib') else 'dev' + so_dest_ompi = 'openmpi' if self.config.config_opts.get('so_to_lib') else 'dev' patterns = [ # Patterns for matching files, format is a tuple as follows: diff --git a/autospec/git.py b/autospec/git.py index 6785164..5e08558 100644 --- a/autospec/git.py +++ b/autospec/git.py @@ -25,12 +25,11 @@ import subprocess import build import buildpattern -import config import tarball from util import call, write_out -def commit_to_git(path): +def commit_to_git(path, config): """Update package's git tree for autospec managed changes.""" call("git init", stdout=subprocess.DEVNULL, cwd=path) diff --git a/autospec/license.py b/autospec/license.py index 0e605f0..ec809c0 100644 --- a/autospec/license.py +++ b/autospec/license.py @@ -28,7 +28,6 @@ import sys import urllib.parse import chardet -import config import download import tarball @@ -41,17 +40,17 @@ license_files = [] hashes = dict() -def process_licenses(lics): +def process_licenses(lics, translations, blacklist): """Handle licenses string from the license server. The license server response may contain multiple space-separated licenses. Add each license individually. """ for lic in lics.split(): - add_license(lic) + add_license(lic, translations, blacklist) -def add_license(lic): +def add_license(lic, translations, blacklist): """Add licenses from the server. Add license from license string lic after checking for duplication or @@ -64,10 +63,10 @@ def add_license(lic): result = False # Translate the license if a translation exists - real_lic_str = config.license_translations.get(lic, lic) + real_lic_str = translations.get(lic, lic) real_lics = real_lic_str.split() for real_lic in real_lics: - if real_lic in config.license_blacklist: + if real_lic in blacklist: continue elif real_lic in licenses: result = True @@ -96,7 +95,7 @@ def decode_license(license): return try_with_charset(license, chardet.detect(license)['encoding']) -def license_from_copying_hash(copying, srcdir): +def license_from_copying_hash(copying, srcdir, config): """Add licenses based on the hash of the copying file.""" try: data = get_contents(copying) @@ -124,7 +123,7 @@ def license_from_copying_hash(copying, srcdir): page = response.decode('utf-8').strip() if page: print("License : ", page, " (server) (", hash_sum, ")") - process_licenses(page) + process_licenses(page, config.license_translations, config.license_blacklist) if page != "none": # Strip the build source directory off the front @@ -139,7 +138,9 @@ def license_from_copying_hash(copying, srcdir): return if hash_sum in config.license_hashes: - add_license(config.license_hashes[hash_sum]) + add_license(config.license_hashes[hash_sum], + config.license_translations, + config.license_blacklist) else: if not config.license_show: return @@ -148,7 +149,7 @@ def license_from_copying_hash(copying, srcdir): print_warning("Visit {0} to enter".format(hash_url)) -def scan_for_licenses(srcdir): +def scan_for_licenses(srcdir, config): """Scan the project directory for things we can use to guess a description and summary.""" targets = ["copyright", "copyright.txt", @@ -167,7 +168,8 @@ def scan_for_licenses(srcdir): for dirpath, dirnames, files in os.walk(srcdir): for name in files: if name.lower() in targets or target_pat.search(name.lower()): - license_from_copying_hash(os.path.join(dirpath, name), srcdir) + license_from_copying_hash(os.path.join(dirpath, name), + srcdir, config) if not licenses: print_fatal(" Cannot find any license or a valid {}.license file!\n".format(tarball.name)) diff --git a/autospec/pkg_integrity.py b/autospec/pkg_integrity.py index 2671269..5f28d57 100644 --- a/autospec/pkg_integrity.py +++ b/autospec/pkg_integrity.py @@ -210,6 +210,7 @@ class Verifier(object): """Set default values.""" self.url = kwargs.get('url', None) self.package_sign_path = kwargs.get('package_sign_path', None) + self.config = kwargs.get('config', None) print(SEPT) @staticmethod @@ -569,13 +570,13 @@ class GPGVerifier(Verifier): EMAIL = get_email(pubkey_loc) sign_status = verify_cli(pubkey_loc, self.package_path, self.package_sign_path) if not sign_status: - if config.old_keyid: - compare_keys(KEYID_TRY, config.old_keyid) + if self.config.old_keyid: + compare_keys(KEYID_TRY, self.config.old_keyid) self.print_result(self.package_path) KEYID = KEYID_TRY - config.signature = self.key_url - config.config_opts['verify_required'] = True - config.rewrite_config_opts(os.path.dirname(self.package_path)) + self.config.signature = self.key_url + self.config.config_opts['verify_required'] = True + self.config.rewrite_config_opts(os.path.dirname(self.package_path)) return True else: self.print_result(False, err_msg=sign_status.strerror) @@ -833,14 +834,17 @@ def apply_verification(verifier, **kwargs): return v.verify() -def from_disk(url, package_path, package_check, interactive=True): +def from_disk(url, package_path, package_check, config, interactive=True): """Run verification.""" verifier = get_verifier(package_path) - return apply_verification(verifier, **{ - 'package_path': package_path, - 'package_check': package_check, - 'url': url, - 'interactive': interactive}) + return apply_verification(verifier, + **{ + 'package_path': package_path, + 'package_check': package_check, + 'url': url, + 'interactive': interactive, + 'config': config, + }) def attempt_verification_per_domain(package_path, url): @@ -879,7 +883,7 @@ def get_integrity_file(package_path): return None -def check(url, download_path, interactive=True): +def check(url, download_path, config, interactive=True): """Run verification based on tar file url.""" package_name = filename_from_url(url) package_path = os.path.join(download_path, package_name) @@ -892,15 +896,15 @@ def check(url, download_path, interactive=True): print_info('Performing package integrity verification') verified = None if package_check is not None: - verified = from_disk(url, package_path, package_check, interactive=interactive) + verified = from_disk(url, package_path, package_check, config, interactive=interactive) elif package_path[-4:] == '.gem': signature_file = get_signature_file(url, download_path) - verified = from_disk(url, package_path, signature_file, interactive=interactive) + verified = from_disk(url, package_path, signature_file, config, interactive=interactive) else: print_info('None of {}.(asc|sig|sign|sha256) is found in {}'.format(package_name, download_path)) signature_file = get_signature_file(url, download_path) if signature_file is not None: - verified = from_disk(url, package_path, signature_file, interactive=interactive) + verified = from_disk(url, package_path, signature_file, config, interactive=interactive) if verified is None: print_info('Unable to find a signature') verified = attempt_verification_per_domain(package_path, url) @@ -936,7 +940,7 @@ def load_specfile(specfile): def main(args): """Verify tar content with signature.""" - from_disk(args.url, args.tar, args.sig) + from_disk(args.url, args.tar, args.sig, config.Config()) if __name__ == '__main__': diff --git a/autospec/pkg_scan.py b/autospec/pkg_scan.py index d232199..89f939a 100644 --- a/autospec/pkg_scan.py +++ b/autospec/pkg_scan.py @@ -17,11 +17,10 @@ # import subprocess -import config import util -def get_whatrequires(pkg): +def get_whatrequires(pkg, yum_conf): """ Write list of packages. @@ -30,7 +29,7 @@ def get_whatrequires(pkg): """ # clean up dnf cache to avoid 'no more mirrors repo' error try: - subprocess.check_output(['dnf', '--config', config.yum_conf, + subprocess.check_output(['dnf', '--config', yum_conf, '--releasever', 'clear', 'clean', 'all']) except subprocess.CalledProcessError as err: util.print_warning("Unable to clean dnf repo: {}, {}".format(pkg, err)) @@ -38,7 +37,7 @@ def get_whatrequires(pkg): try: out = subprocess.check_output(['dnf', 'repoquery', - '--config', config.yum_conf, + '--config', yum_conf, '--releasever', 'clear', '--archlist=src', '--recursive', '--queryformat=%{NAME}', '--whatrequires', pkg]).decode('utf-8') diff --git a/autospec/specdescription.py b/autospec/specdescription.py index 31eb72b..3420161 100644 --- a/autospec/specdescription.py +++ b/autospec/specdescription.py @@ -29,7 +29,6 @@ import os import re -import config import license import util @@ -84,7 +83,7 @@ def assign_description(description, score): default_description_score = score -def description_from_spec(specfile): +def description_from_spec(specfile, translations, blacklist): """Parse any existing RPM specfiles.""" try: with util.open_auto(specfile, 'r') as specfd: @@ -105,15 +104,15 @@ def description_from_spec(specfile): if line.startswith("License:") and not any(e in line for e in excludes): splits = line.split(":")[1:] words = ":".join(splits).strip() - if words in config.license_translations: + if words in translations: print("Adding license from spec:", words) - license.add_license(words) + license.add_license(words, translations, blacklist) else: words = clean_license_string(words).split() for word in words: if ":" not in word and not word.startswith("@"): print("Adding license from spec:", word) - license.add_license(word) + license.add_license(word, translations, blacklist) if line.startswith("Summary: "): assign_summary(line[9:], 4) @@ -128,7 +127,7 @@ def description_from_spec(specfile): assign_description(specdesc, 4) -def description_from_pkginfo(pkginfo): +def description_from_pkginfo(pkginfo, translations, blacklist): """Parse existing package info files.""" try: with util.open_auto(pkginfo, 'r') as pkgfd: @@ -146,15 +145,15 @@ def description_from_pkginfo(pkginfo): if line.lower().startswith("license:") and not any(e in line for e in excludes): splits = line.split(":")[1:] words = ":".join(splits).strip() - if words in config.license_translations: + if words in translations: print("Adding license from PKG-INFO:", words) - license.add_license(words) + license.add_license(words, translations, blacklist) else: words = clean_license_string(words).split() for word in words: if ":" not in word: print("Adding license from PKG-INFO:", word) - license.add_license(word) + license.add_license(word, translations, blacklist) for sub in ["Summary: ", "abstract: "]: if line.startswith(sub): @@ -241,7 +240,7 @@ def description_from_readme(readmefile): assign_description(desc, score) -def scan_for_description(package, dirn): +def scan_for_description(package, dirn, translations, blacklist): """Scan the project directory for things we can use to guess a description and summary.""" test_pat = re.compile(r"tests?") dirpath_seen = "" @@ -253,13 +252,13 @@ def scan_for_description(package, dirn): if name.lower().endswith(".pdf"): continue if name.lower().endswith(".spec"): - description_from_spec(os.path.join(dirpath, name)) + description_from_spec(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith("pkg-info"): - description_from_pkginfo(os.path.join(dirpath, name)) + description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith("meta.yml"): - description_from_pkginfo(os.path.join(dirpath, name)) + description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith("description"): - description_from_pkginfo(os.path.join(dirpath, name)) + description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith(".pc"): summary_from_pkgconfig(os.path.join(dirpath, name), package) if name.startswith("DESCRIPTION"): @@ -272,13 +271,13 @@ def scan_for_description(package, dirn): print("Summary :", default_summary.strip()) -def load_specfile(specfile): +def load_specfile(specfile, description, summary): """Load specfile with parse results.""" - if config.custom_desc: - specfile.default_desc = "\n".join(config.custom_desc) + if description: + specfile.default_desc = "\n".join(description) else: specfile.default_desc = default_description - if config.custom_summ: - specfile.default_sum = config.custom_summ[0] + if summary: + specfile.default_sum = summary[0] else: specfile.default_sum = default_summary diff --git a/autospec/specfiles.py b/autospec/specfiles.py index 4a96d47..5b42460 100644 --- a/autospec/specfiles.py +++ b/autospec/specfiles.py @@ -26,7 +26,6 @@ import types from collections import OrderedDict import buildreq -import config import tarball from util import _file_write from util import open_auto @@ -35,12 +34,13 @@ from util import open_auto class Specfile(object): """Holds data and methods needed to write the spec file.""" - def __init__(self, url, version, name, release): + def __init__(self, url, version, name, release, config): """Add default information for specfile template.""" self.url = url self.version = version self.name = name self.release = release + self.config = config self.keepstatic = False self.urlban = "" self.no_autostart = False @@ -179,9 +179,9 @@ class Specfile(object): self._write("Source{0} : {1}\n".format(count, source)) # if package is verified, include the signature in the source tarball - if self.keyid and config.signature: + if self.keyid and self.config.signature: count += 1 - self._write_strip(f"Source{count} : {config.signature}") + self._write_strip(f"Source{count} : {self.config.signature}") for source in self.extra_sources: count += 1 @@ -224,13 +224,13 @@ class Specfile(object): def write_strip_command(self): """Write commands to prevent stripping binary if requested.""" - if config.config_opts['nostrip']: + if self.config.config_opts['nostrip']: self._write("# Suppress stripping binaries\n") self._write("%define __strip /bin/true\n%define debug_package %{nil}\n") def write_debug_command(self): """Write commands to prevent debug info generation if requested.""" - if config.config_opts['nodebug']: + if self.config.config_opts['nodebug']: self._write("# Suppress generation of debuginfo\n") self._write("%global debug_package %{nil}\n") @@ -243,8 +243,8 @@ class Specfile(object): for count, patch in enumerate(self.patches): self._write("Patch{0}: {1}\n".format(count + 1, patch.split()[0])) # Write the version-specific patches - for version in config.verpatches: - for count, patch in enumerate(config.verpatches[version], start=count + 1): + for version in self.config.verpatches: + for count, patch in enumerate(self.config.verpatches[version], start=count + 1): self._write("Patch{0}: {1}\n".format(count + 1, patch.split()[0])) def write_description(self): @@ -270,9 +270,9 @@ class Specfile(object): deps["libexec"] = ["config", "license"] deps["lib32"] = ["data", "license"] deps["python"] = ["python3"] - if config.config_opts['dev_requires_extras']: + if self.config.config_opts.get('dev_requires_extras'): deps["dev"].append("extras") - if config.config_opts['openmpi']: + if self.config.config_opts.get('openmpi'): deps["dev"].append("openmpi") for k, v in self.custom_extras.items(): if "requires" in v: @@ -355,7 +355,7 @@ class Specfile(object): if pkg in ["ignore", "main", "locales"]: continue for script in ["post", "pre"]: - content = config.read_conf_file("{}.{}".format(script, pkg)) + content = self.config.read_conf_file("{}.{}".format(script, pkg)) if content: self._write("\n%{0} {1}\n".format(script, pkg)) content = ['{}\n'.format(l) for l in content] @@ -401,7 +401,7 @@ class Specfile(object): if export_epoch: # time.time() returns a float, but we only need second-precision self._write_strip("export SOURCE_DATE_EPOCH={}".format(int(time.time()))) - if config.config_opts['asneeded']: + if self.config.config_opts['asneeded']: self._write_strip("unset LD_AS_NEEDED\n") def write_proxy_exports(self): @@ -418,9 +418,9 @@ class Specfile(object): self._write_strip("{}\n".format(line)) self._write_strip("## make_prepend end") if build32: - self._write_strip("make {} {} {}".format(config.parallel_build, self.extra_make, self.extra32_make)) + self._write_strip("make {} {} {}".format(self.config.parallel_build, self.extra_make, self.extra32_make)) else: - self._write_strip("make {} {}".format(config.parallel_build, self.extra_make)) + self._write_strip("make {} {}".format(self.config.parallel_build, self.extra_make)) def write_install_openmpi(self): """Write make install line (openmpi) to spec file.""" @@ -533,19 +533,19 @@ class Specfile(object): destination)) self.apply_patches() if self.default_pattern != 'cmake': - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ..") self._write_strip("cp -a {} build32".format(self.tarball_prefix)) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ..") self._write_strip("cp -a {} buildavx2".format(self.tarball_prefix)) self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ..") self._write_strip("cp -a {} buildavx512".format(self.tarball_prefix)) self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd ..") self._write_strip("cp -a {} build-openmpi".format(self.tarball_prefix)) self._write_strip("popd") @@ -570,7 +570,7 @@ class Specfile(object): # a source of headaches for downstream users. self._write_strip("export GCC_IGNORE_WERROR=1\n") - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: self._write_strip("export CC=clang\n") self._write_strip("export CXX=clang++\n") self._write_strip("export LD=ld.gold\n") @@ -583,12 +583,12 @@ class Specfile(object): if not self.set_gopath: self._write_strip("export GOPROXY=file:///usr/share/goproxy") - if config.config_opts['optimize_size']: - if config.config_opts['use_clang']: + if self.config.config_opts['optimize_size']: + if self.config.config_opts['use_clang']: flags.extend(["-Os", "-ffunction-sections", "-fdata-sections"]) else: flags.extend(["-Os", "-ffunction-sections", "-fdata-sections", "-fno-semantic-interposition"]) - if config.config_opts['security_sensitive']: + if self.config.config_opts['security_sensitive']: flags.append("-fstack-protector-strong") if arch == 'x86_64': flags.append("-mzero-caller-saved-regs=used") @@ -596,10 +596,10 @@ class Specfile(object): flags.extend(["-O3", "-march=haswell"]) if self.need_avx512_flags: flags.extend(["-O3", "-march=skylake-avx512"]) - if config.config_opts['insecure_build']: + if self.config.config_opts['insecure_build']: self._write_strip('export CFLAGS="-O3 -g -fopt-info-vec "\n') self._write_strip("unset LDFLAGS\n") - if config.config_opts['conservative_flags']: + if self.config.config_opts['conservative_flags']: self._write_strip('export CFLAGS="-O2 -g -Wp,-D_FORTIFY_SOURCE=2 ' "-fexceptions -fstack-protector " "--param=ssp-buffer-size=32 -Wformat " @@ -608,17 +608,17 @@ class Specfile(object): '-march=westmere -mtune=haswell"\n') self._write_strip("export CXXFLAGS=$CFLAGS\n") self._write_strip("unset LDFLAGS\n") - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: self._write_strip("unset LDFLAGS\n") - if config.config_opts['funroll-loops']: - if config.config_opts['use_clang']: + if self.config.config_opts['funroll-loops']: + if self.config.config_opts['use_clang']: flags.extend(["-O3"]) else: flags.extend(["-O3", "-fno-semantic-interposition", "-falign-functions=32", "-fno-math-errno", "-fno-trapping-math"]) if self.default_pattern != 'qmake': - if config.config_opts['use_lto']: + if self.config.config_opts['use_lto']: flags.extend(["-O3", lto, "-ffat-lto-objects"]) - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: self._write_strip("export AR=llvm-ar\n") self._write_strip("export RANLIB=llvm-ranlib\n") self._write_strip("export NM=llvm-nm\n") @@ -628,26 +628,26 @@ class Specfile(object): self._write_strip("export NM=gcc-nm\n") else: flags.extend(["-fno-lto"]) - if config.config_opts['fast-math']: + if self.config.config_opts['fast-math']: flags.extend(["-ffast-math", "-ftree-loop-vectorize"]) - if config.config_opts['pgo']: + if self.config.config_opts['pgo']: flags.extend(["-O3"]) if self.gcov_file: flags = list(filter((lto).__ne__, flags)) flags.extend(["-O3", "-fauto-profile=%{{SOURCE{0}}}".format(self.source_index[self.sources["gcov"][0]])]) - if flags or config.config_opts['broken_c++']: + if flags or self.config.config_opts['broken_c++']: flags = sorted(list(set(flags))) self._write_strip('export CFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags))) self._write_strip('export FCFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags))) self._write_strip('export FFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags))) # leave the export CXXFLAGS line open in case self._write('export CXXFLAGS="$CXXFLAGS {0} '.format(" ".join(flags))) - if config.config_opts['broken_c++']: + if self.config.config_opts['broken_c++']: self._write('-std=gnu++98') # close the open quote from CXXFLAGS export and add newline self._write('"\n') - if config.profile_payload and config.profile_payload[0] and not self.need_avx2_flags: + if self.config.profile_payload and self.config.profile_payload[0] and not self.need_avx2_flags: genflags = [] useflags = [] genflags.extend(["-fprofile-generate", "-fprofile-dir=/var/tmp/pgo", "-fprofile-update=atomic"]) @@ -667,7 +667,7 @@ class Specfile(object): def write_check(self): """Write check section to spec file.""" - if self.tests_config and not config.config_opts['skip_tests']: + if self.tests_config and not self.config.config_opts['skip_tests']: self._write_strip("%check") self._write_strip("export LANG=C.UTF-8") self.write_proxy_exports() @@ -685,7 +685,7 @@ class Specfile(object): def write_profile_payload(self, pattern=None): """Write the profile_payload specified for this package.""" - if not config.profile_payload: + if not self.config.profile_payload: return use_subdir = True init = "" @@ -694,20 +694,20 @@ class Specfile(object): init = f"{self.get_profile_generate_flags()}" \ f"%configure " \ f"{self.disable_static} " \ - f"{config.extra_configure} " \ - f"{config.extra_configure64}" + f"{self.config.extra_configure} " \ + f"{self.config.extra_configure64}" elif pattern == "configure_ac": init = f"{self.get_profile_generate_flags()}" \ f"%reconfigure " \ f"{self.disable_static} " \ - f"{config.extra_configure} " \ - f"{config.extra_configure64}" + f"{self.config.extra_configure} " \ + f"{self.config.extra_configure64}" elif pattern == "autogen": init = f"{self.get_profile_generate_flags()}" \ f"%autogen " \ f"{self.disable_static} " \ - f"{config.extra_configure} " \ - f"{config.extra_configure64}" + f"{self.config.extra_configure} " \ + f"{self.config.extra_configure64}" elif pattern == "cmake": use_subdir = False init = f"{self.get_profile_generate_flags()}" @@ -723,7 +723,7 @@ class Specfile(object): if use_subdir and self.subdir: self._write_strip("popd") self._write_strip("\n") - self._write_strip("\n".join(config.profile_payload)) + self._write_strip("\n".join(self.config.profile_payload)) self._write_strip("\nmake clean\n") if post: self._write_strip(post) @@ -738,7 +738,7 @@ class Specfile(object): self.write_license_files() - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self._write_strip("%make_install32 {} {}".format(self.extra_make_install, self.extra_make32_install)) @@ -750,17 +750,17 @@ class Specfile(object): self._write_strip("fi") self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ../buildavx512/" + self.subdir) self._write_strip("%s_avx512 %s\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2/" + self.subdir) self._write_strip("%s_avx2 %s\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd ../build-openmpi/" + self.subdir) self.write_install_openmpi() self._write_strip("popd") @@ -1004,7 +1004,7 @@ class Specfile(object): if self.subdir: self._write_strip("pushd " + self.subdir) - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd clr-build32") self._write_strip("%make_install32 {} {}".format(self.extra_make_install, self.extra_make32_install)) @@ -1016,17 +1016,17 @@ class Specfile(object): self._write_strip("fi") self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd clr-build-avx512") self._write_strip("%s_avx512 %s || :\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd clr-build-avx2") self._write_strip("%s_avx2 %s || :\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd clr-build-openmpi") self.write_install_openmpi() self._write_strip("popd") @@ -1040,8 +1040,7 @@ class Specfile(object): self.write_find_lang() - @staticmethod - def get_profile_generate_flags(): + def get_profile_generate_flags(self): """Return profile generate flags if proper configuration is set. If config.profile_payload is non-empty, returns @@ -1053,7 +1052,7 @@ class Specfile(object): otherwise an empty string is returned. """ - if config.profile_payload and config.profile_payload[0]: + if self.config.profile_payload and self.config.profile_payload[0]: return 'CFLAGS="${CFLAGS_GENERATE}" ' \ 'CXXFLAGS="${CXXFLAGS_GENERATE}" ' \ 'FFLAGS="${FFLAGS_GENERATE}" ' \ @@ -1061,8 +1060,7 @@ class Specfile(object): 'LDFLAGS="${LDFLAGS_GENERATE}" ' return "" - @staticmethod - def get_profile_use_flags(): + def get_profile_use_flags(self): """Return profile generate flags if proper configuration is set. If config.profile_payload is non-empty, returns @@ -1074,7 +1072,7 @@ class Specfile(object): otherwise an empty string is returned. """ - if config.profile_payload and config.profile_payload[0]: + if self.config.profile_payload and self.config.profile_payload[0]: return 'CFLAGS="${CFLAGS_USE}" ' \ 'CXXFLAGS="${CXXFLAGS_USE}" ' \ 'FFLAGS="${FFLAGS_USE}" ' \ @@ -1118,13 +1116,13 @@ class Specfile(object): self._write_strip("{0}%configure {1} {2} {3}" .format(self.get_profile_use_flags(), self.disable_static, - config.extra_configure, - config.extra_configure64)) + self.config.extra_configure, + self.config.extra_configure64)) self.write_make_line() if self.subdir: self._write_strip("popd") self._write_strip("\n") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_build_prepend() self.write_32bit_exports() @@ -1134,12 +1132,12 @@ class Specfile(object): "--host=i686-generic-linux-gnu " "--target=i686-clr-linux-gnu" .format(self.disable_static, - config.extra_configure, - config.extra_configure32)) + self.config.extra_configure, + self.config.extra_configure32)) self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx2/" + self.subdir) self.write_build_prepend() @@ -1148,12 +1146,12 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self._write_strip("%configure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx2)) + self.config.extra_configure, + self.config.extra_configure_avx2)) self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx512/" + self.subdir) self.write_build_prepend() @@ -1162,12 +1160,12 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=skylake-avx512\"") self._write_strip("%configure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx512)) + self.config.extra_configure, + self.config.extra_configure_avx512)) self.write_make_line() self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd ../build-openmpi/" + self.subdir) self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh") self._write_strip("module load openmpi") @@ -1178,9 +1176,9 @@ class Specfile(object): self._write_strip("export FFLAGS=\"$FFLAGS -m64 -march=haswell\"") self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self._write_strip("./configure {0} \\\n{1} {2}" - .format(config.conf_args_openmpi, + .format(self.config.conf_args_openmpi, self.disable_static, - config.extra_configure_openmpi)) + self.config.extra_configure_openmpi)) self.write_make_line() self._write_strip("module unload openmpi") self._write_strip("popd") @@ -1199,12 +1197,12 @@ class Specfile(object): self._write_strip("{0}%reconfigure {1} {2} {3}" .format(self.get_profile_use_flags(), self.disable_static, - config.extra_configure, - config.extra_configure64)) + self.config.extra_configure, + self.config.extra_configure64)) self.write_make_line() if self.subdir: self._write_strip("popd") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_build_prepend() self.write_32bit_exports() @@ -1214,12 +1212,12 @@ class Specfile(object): "--host=i686-generic-linux-gnu " "--target=i686-clr-linux-gnu" .format(self.disable_static, - config.extra_configure, - config.extra_configure32)) + self.config.extra_configure, + self.config.extra_configure32)) self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx2/" + self.subdir) self.write_build_prepend() @@ -1228,12 +1226,12 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self._write_strip("%reconfigure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx2)) + self.config.extra_configure, + self.config.extra_configure_avx2)) self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx512/" + self.subdir) self.write_build_prepend() @@ -1242,8 +1240,8 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=skylake-avx512\"") self._write_strip("%reconfigure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx512)) + self.config.extra_configure, + self.config.extra_configure_avx512)) self.write_make_line() self._write_strip("popd") @@ -1263,12 +1261,12 @@ class Specfile(object): if self.subdir: self._write_strip("popd") self._write_strip("\n") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_32bit_exports() self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2" + self.subdir) self.write_build_prepend() self._write_strip("export CFLAGS=\"$CFLAGS -m64 -march=haswell\"") @@ -1276,7 +1274,7 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ../buildavx512" + self.subdir) self.write_build_prepend() self._write_strip("export CFLAGS=\"$CFLAGS -m64 -march=skylake-avx512 -mprefer-vector-width=512\"") @@ -1298,11 +1296,11 @@ class Specfile(object): self._write_strip("{0}%autogen {1} {2} {3}" .format(self.get_profile_use_flags(), self.disable_static, - config.extra_configure, - config.extra_configure64)) + self.config.extra_configure, + self.config.extra_configure64)) self.write_make_line() self._write_strip("\n") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_build_prepend() self.write_32bit_exports() @@ -1312,12 +1310,12 @@ class Specfile(object): "--host=i686-generic-linux-gnu " "--target=i686-clr-linux-gnu" .format(self.disable_static, - config.extra_configure, - config.extra_configure32)) + self.config.extra_configure, + self.config.extra_configure32)) self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2/" + self.subdir) self.write_build_prepend() self._write_strip('export CFLAGS="$CFLAGS -m64 -march=haswell "') @@ -1325,12 +1323,12 @@ class Specfile(object): self._write_strip('export LDFLAGS="$LDFLAGS -m64 -march=haswell "') self._write_strip("%autogen {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx2)) + self.config.extra_configure, + self.config.extra_configure_avx2)) self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ../buildavx512/" + self.subdir) self.write_build_prepend() self._write_strip('export CFLAGS="$CFLAGS -m64 -march=skylake-avx512 "') @@ -1338,8 +1336,8 @@ class Specfile(object): self._write_strip('export LDFLAGS="$LDFLAGS -m64 -march=skylake-avx512 "') self._write_strip("%autogen {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx512)) + self.config.extra_configure, + self.config.extra_configure_avx512)) self.write_make_line() self._write_strip("popd") @@ -1354,9 +1352,9 @@ class Specfile(object): self._write_strip("export MAKEFLAGS=%{?_smp_mflags}") if self.subdir: self._write_strip("pushd " + self.subdir) - self._write_strip("python3 setup.py build " + config.extra_configure) + self._write_strip("python3 setup.py build " + self.config.extra_configure) self._write_strip("\n") - if self.tests_config and not config.config_opts['skip_tests']: + if self.tests_config and not self.config.config_opts['skip_tests']: self._write_strip("%check") # Prevent setuptools from hitting the internet self.write_proxy_exports() @@ -1388,9 +1386,9 @@ class Specfile(object): self.write_variables() if self.subdir: self._write_strip("pushd " + self.subdir) - self._write_strip("python3.6 setup.py build -b py3 " + config.extra_configure) + self._write_strip("python3.6 setup.py build -b py3 " + self.config.extra_configure) self._write_strip("\n") - if self.tests_config and not config.config_opts['skip_tests']: + if self.tests_config and not self.config.config_opts['skip_tests']: self._write_strip("%check") # Prevent setuptools from hitting the internet self.write_proxy_exports() @@ -1535,7 +1533,7 @@ class Specfile(object): self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("mkdir -p clr-build-avx2") self._write_strip("pushd clr-build-avx2") saved_avx2flags = self.need_avx2_flags @@ -1549,7 +1547,7 @@ class Specfile(object): self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("mkdir -p clr-build-avx512") self._write_strip("pushd clr-build-avx512") saved_avx512flags = self.need_avx512_flags @@ -1563,7 +1561,7 @@ class Specfile(object): self.write_make_line() self._write_strip("popd") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("mkdir -p clr-build32") self._write_strip("pushd clr-build32") self.write_build_prepend() @@ -1577,7 +1575,7 @@ class Specfile(object): self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("mkdir -p clr-build-openmpi") self._write_strip("pushd clr-build-openmpi") self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh") @@ -1607,9 +1605,9 @@ class Specfile(object): def write_qmake_pattern(self): """Write qmake build pattern to spec file.""" extra_qmake_args = "" - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: extra_qmake_args = "-spec linux-clang " - if config.config_opts['use_lto']: + if self.config.config_opts['use_lto']: extra_qmake_args += "-config ltcg -config fat-static-lto " else: extra_qmake_args += "QMAKE_CFLAGS+=-fno-lto QMAKE_CXXFLAGS+=-fno-lto " @@ -1624,18 +1622,18 @@ class Specfile(object): if self.subdir: self._write_strip("pushd " + self.subdir) - self._write_strip("%qmake {} {}".format(extra_qmake_args, config.extra_configure)) + self._write_strip("%qmake {} {}".format(extra_qmake_args, self.config.extra_configure)) self._write_strip("test -r config.log && cat config.log") self.write_make_line() if self.subdir: self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2/" + self.subdir) self._write("%qmake 'QT_CPU_FEATURES.x86_64 += avx avx2 bmi bmi2 f16c fma lzcnt popcnt'\\\n") self._write(" QMAKE_CFLAGS+=-march=haswell QMAKE_CXXFLAGS+=-march=haswell \\\n") - self._write(" QMAKE_LFLAGS+=-march=haswell {} {}\n".format(extra_qmake_args, config.extra_configure)) + self._write(" QMAKE_LFLAGS+=-march=haswell {} {}\n".format(extra_qmake_args, self.config.extra_configure)) self.write_make_line() self._write_strip("popd") @@ -1708,7 +1706,7 @@ class Specfile(object): self.write_proxy_exports() self._write_strip("export LANG=C.UTF-8") self.write_variables() - self._write_strip("scons{} {}".format(config.parallel_build, config.extra_configure)) + self._write_strip("scons{} {}".format(self.config.parallel_build, self.config.extra_configure)) self.write_build_append() self._write_strip("\n") self._write_strip("%install") @@ -1853,23 +1851,23 @@ class Specfile(object): if self.subdir: self._write_strip("pushd " + self.subdir) self._write_strip('CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" LDFLAGS="$LDFLAGS" meson --libdir=lib64 --prefix=/usr --buildtype=plain {0} {1} builddir' - .format(config.extra_configure, - config.extra_configure64)) + .format(self.config.extra_configure, + self.config.extra_configure64)) self._write_strip("ninja -v -C builddir") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip('CFLAGS="$CFLAGS -m64 -march=haswell" CXXFLAGS="$CXXFLAGS -m64 -march=haswell " LDFLAGS="$LDFLAGS -m64 -march=haswell" ' 'meson --libdir=lib64/haswell --prefix=/usr --buildtype=plain {0} ' - '{1} builddiravx2'.format(config.extra_configure, config.extra_configure64)) + '{1} builddiravx2'.format(self.config.extra_configure, self.config.extra_configure64)) self._write_strip('ninja -v -C builddiravx2') if self.subdir: self._write_strip("popd") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_32bit_exports() self._write_strip('meson ' '--libdir=lib32 --prefix=/usr --buildtype=plain {0} {1} builddir' - .format(config.extra_configure, - config.extra_configure32)) + .format(self.config.extra_configure, + self.config.extra_configure32)) self._write_strip('ninja -v -C builddir') self._write_strip('popd') @@ -1879,7 +1877,7 @@ class Specfile(object): self._write_strip("%install") self.write_install_prepend() self.write_license_files() - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip('pushd ../build32/' + self.subdir) self._write_strip('DESTDIR=%{buildroot} ninja -C builddir install') self._write_strip("if [ -d %{buildroot}/usr/lib32/pkgconfig ]") @@ -1891,7 +1889,7 @@ class Specfile(object): self._write_strip("popd") if self.subdir: self._write_strip("pushd " + self.subdir) - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip('DESTDIR=%{buildroot} ninja -C builddiravx2 install') self._write_strip("DESTDIR=%{buildroot} ninja -C builddir install") @@ -1951,7 +1949,7 @@ class Specfile(object): # Write version-specific patch commands for version in self.verpatches: if self.verpatches[version]: - self._write("cd ../{}\n".format(self.build_dirs[config.versions[version]])) + self._write("cd ../{}\n".format(self.build_dirs[self.config.versions[version]])) for p in self.verpatches[version]: name = p.split(None, 1)[0] if name == p: diff --git a/autospec/tarball.py b/autospec/tarball.py index 075d077..b426314 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -27,7 +27,6 @@ from collections import OrderedDict import build import buildpattern import buildreq -import config import download from util import call, do_regex, get_sha1sum, print_fatal, write_out @@ -266,7 +265,7 @@ def detect_build_from_url(url): buildpattern.set_build_pattern("phpize", 10) -def set_multi_version(ver): +def set_multi_version(ver, config): """Add ver to multi_version set and return latest version.""" global multi_version @@ -287,7 +286,7 @@ def set_multi_version(ver): return latest -def name_and_version(name_arg, version_arg, filemanager): +def name_and_version(name_arg, version_arg, filemanager, config): """Parse the url for the package name and version.""" global rawname global url @@ -307,7 +306,7 @@ def name_and_version(name_arg, version_arg, filemanager): # rawname == name in this case name = name_arg rawname = name_arg - version = set_multi_version(version_arg) + version = set_multi_version(version_arg, config) return name, rawname, convert_version(version, name) name = name_arg @@ -439,7 +438,7 @@ def name_and_version(name_arg, version_arg, filemanager): # override name and version from commandline name = name_arg if name_arg else name version = version_arg if version_arg else version - version = set_multi_version(version) + version = set_multi_version(version, config) return name, rawname, convert_version(version, name) @@ -474,7 +473,7 @@ def process_go_archives(go_archives): buildpattern.sources["godep"] += [url_info, url_mod, url_zip] -def process_multiver_archives(main_src, multiver_archives): +def process_multiver_archives(main_src, multiver_archives, config): """Set up multiversion archives.""" config_versions = config.parse_config_versions(build.download_path) # Check if exist more than one version. @@ -485,10 +484,10 @@ def process_multiver_archives(main_src, multiver_archives): buildpattern.sources["version"].append(extraurl) multiver_archives.append(extraurl) multiver_archives.append('') - set_multi_version(None) + set_multi_version(None, config) -def process_archives(main_src, archives): +def process_archives(main_src, archives, config): """Process extra sources needed by package. This sources include: archives, go archives and multiversion. @@ -502,7 +501,7 @@ def process_archives(main_src, archives): process_go_archives(go_archives) # Add multiversion for the rest of the patterns else: - process_multiver_archives(main_src, multiver_archives) + process_multiver_archives(main_src, multiver_archives, config) full_archives = archives + go_archives + multiver_archives # Download and extract full list @@ -534,7 +533,7 @@ def extract_sources(main_src, archives_src): src.extract() -def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): +def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager, config): """Download and process the tarball at url_arg.""" global url global name @@ -552,7 +551,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): # Create the download path for content and set build.download_path create_download_path(target) # determine name and version of package - name, rawname, version = name_and_version(name_arg, ver_arg, filemanager) + name, rawname, version = name_and_version(name_arg, ver_arg, filemanager, config) # Store the top-level version config.versions[version] = url # set gcov file information, must be done after name is set since the gcov @@ -570,7 +569,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): print_header() # Download and process extra sources: archives, go archives and # multiversion - archives_src = process_archives(main_src, archives) + archives_src = process_archives(main_src, archives, config) # Extract all sources extract_sources(main_src, archives_src) diff --git a/tests/test_build.py b/tests/test_build.py index 8016152..d466afd 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,18 +3,12 @@ import tempfile import os from unittest.mock import patch, mock_open, MagicMock import build +import config import files class TestBuildpattern(unittest.TestCase): - @classmethod - def setUpClass(self): - """ - Class setup method to configure necessary modules - """ - build.config.setup_patterns() - def setUp(self): """ Test setup method to reset the buildpattern module @@ -25,7 +19,6 @@ class TestBuildpattern(unittest.TestCase): build.base_path = None build.download_path = None build.buildreq.buildreqs = set() - build.config.config_opts['32bit'] = False def test_setup_workingdir(self): """ @@ -40,10 +33,10 @@ class TestBuildpattern(unittest.TestCase): """ Test simple_pattern_pkgconfig with match """ - build.buildreq.config.config_opts['32bit'] = False build.simple_pattern_pkgconfig('line to test for testpkg.xyz', r'testpkg.xyz', - 'testpkg') + 'testpkg', + False) self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs) self.assertEqual(build.must_restart, 1) @@ -51,10 +44,10 @@ class TestBuildpattern(unittest.TestCase): """ Test simple_pattern_pkgconfig with match and 32bit option set """ - build.buildreq.config.config_opts['32bit'] = True build.simple_pattern_pkgconfig('line to test for testpkg.zyx', r'testpkg.zyx', - 'testpkgz') + 'testpkgz', + True) self.assertIn('pkgconfig(32testpkgz)', build.buildreq.buildreqs) self.assertIn('pkgconfig(testpkgz)', build.buildreq.buildreqs) self.assertEqual(build.must_restart, 1) @@ -65,7 +58,8 @@ class TestBuildpattern(unittest.TestCase): """ build.simple_pattern_pkgconfig('line to test for somepkg.xyz', r'testpkg.xyz', - 'testpkg') + 'testpkg', + False) self.assertEqual(build.buildreq.buildreqs, set()) self.assertEqual(build.must_restart, 0) @@ -95,7 +89,8 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with no match """ - build.failed_pattern('line to test for failure: somepkg', r'(test)', 0) + conf = config.Config() + build.failed_pattern('line to test for failure: somepkg', conf, r'(test)', 0) self.assertEqual(build.buildreq.buildreqs, set()) self.assertEqual(build.must_restart, 0) @@ -104,7 +99,8 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool unset and initial match, but no match in failed_commands. """ - build.failed_pattern('line to test for failure: testpkg', r'(test)', 0) + conf = config.Config() + build.failed_pattern('line to test for failure: testpkg', conf, r'(test)', 0) self.assertEqual(build.buildreq.buildreqs, set()) self.assertEqual(build.must_restart, 0) @@ -112,7 +108,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool unset and match in failed_commands """ - build.failed_pattern('line to test for failure: lex', r'(lex)', 0) + conf = config.Config() + conf.setup_patterns() + build.failed_pattern('line to test for failure: lex', conf, r'(lex)', 0) self.assertIn('flex', build.buildreq.buildreqs) self.assertEqual(build.must_restart, 1) @@ -120,8 +118,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to pkgconfig """ - build.buildreq.config.config_opts['32bit'] = False + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.xyz', + conf, r'(testpkg)', 0, # verbose=0 buildtool='pkgconfig') @@ -132,7 +131,10 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to R """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: testpkg.r', + conf, r'(testpkg)', 0, # verbose=0 buildtool='R') @@ -144,7 +146,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to perl """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.pl', + conf, r'(testpkg)', 0, # verbose=0 buildtool='perl') @@ -155,7 +159,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to pypi """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.py', + conf, r'(testpkg)', 0, # verbose=0 buildtool='pypi') @@ -167,7 +173,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to ruby, but no match in config.gems, it should just prepend 'rubygem-' to the package name. """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.rb', + conf, r'(testpkg)', 0, # verbose=0 buildtool='ruby') @@ -180,7 +188,10 @@ class TestBuildpattern(unittest.TestCase): config.gems. In the particular case of test/unit, the result should be rubygem-test-unit. """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: test/unit', + conf, r'(test/unit)', 0, # verbose=0 buildtool='ruby') @@ -192,7 +203,10 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to ruby table and a match in config.gems """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: test/unit', + conf, r'(test/unit)', 0, # verbose=0 buildtool='ruby table') @@ -204,7 +218,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to ruby table but no match in config.gems. This should not modify anything. """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg', + conf, r'(testpkg)', 0, # verbose=0 buildtool='ruby table') @@ -216,7 +232,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to maven, but no match in config.maven_jars, it should just prepend 'mvn-' to the package name. """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg', + conf, r'(testpkg)', 0, # verbose=0 buildtool='maven') @@ -229,7 +247,10 @@ class TestBuildpattern(unittest.TestCase): config.maven_jars. In the particular case of aether, the corresponding maven jar is 'mvn-aether-core' """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: aether', + conf, r'(aether)', 0, # verbose=0 buildtool='maven') @@ -244,8 +265,6 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -269,8 +288,6 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -295,8 +312,6 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -320,18 +335,19 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True + conf = config.Config() + conf.setup_patterns() + conf.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\nwhich: no qmake\nexiting' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup @@ -347,17 +363,18 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\nchecking for Apache test module support\nexiting' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup @@ -369,11 +386,12 @@ class TestBuildpattern(unittest.TestCase): Test parse_build_results with a test log indicating failure due to a missing package. """ - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call open_auto_backup = build.util.open_auto build.util.call = MagicMock(return_value=None) - fm = files.FileManager() + fm = files.FileManager(conf) with open('tests/builderrors', 'r') as f: builderrors = f.readlines() @@ -382,7 +400,7 @@ class TestBuildpattern(unittest.TestCase): input, output = error.strip('\n').split('|') build.buildreq.buildreqs = set() build.util.open_auto = mock_open(read_data=input) - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) self.assertIn(output, build.buildreq.buildreqs) self.assertGreater(build.must_restart, 0) @@ -398,10 +416,11 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\n' \ @@ -414,7 +433,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup @@ -432,10 +451,11 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\n' \ @@ -450,7 +470,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup diff --git a/tests/test_buildreq.py b/tests/test_buildreq.py index d78298e..09d23e3 100644 --- a/tests/test_buildreq.py +++ b/tests/test_buildreq.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, mock_open, patch import json import io import buildreq +import config class TestBuildreq(unittest.TestCase): @@ -15,7 +16,6 @@ class TestBuildreq(unittest.TestCase): Class setup method to configure necessary modules """ buildreq.banned_buildreqs.add('bannedreq') - buildreq.config.setup_patterns() def setUp(self): """ @@ -26,9 +26,6 @@ class TestBuildreq(unittest.TestCase): buildreq.requires = set() buildreq.verbose = False buildreq.cargo_bin = False - buildreq.config.config_opts['32bit'] = False - buildreq.config.cmake_modules = {} - buildreq.config.os_packages = set() buildreq.buildpattern.pattern_strength = 0 def test_add_buildreq(self): @@ -53,29 +50,28 @@ class TestBuildreq(unittest.TestCase): buildreqs but not yet present in requires """ buildreq.add_buildreq('testreq') - self.assertTrue(buildreq.add_requires('testreq')) + self.assertTrue(buildreq.add_requires('testreq', ['testreq'])) self.assertIn('testreq', buildreq.requires) def test_add_requires_not_in_buildreqs(self): """ Test add_requires with unbanned new req not present in buildreqs. """ - self.assertFalse(buildreq.add_requires('testreq')) + self.assertFalse(buildreq.add_requires('testreq', [])) self.assertNotIn('testreq', buildreq.requires) def test_add_pkgconfig_buildreq(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to False """ - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq')) + self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', False)) self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) def test_add_pkgconfig_buildreq_32bit(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to True """ - buildreq.config.config_opts['32bit'] = True - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq')) + self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', True)) self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) self.assertIn('pkgconfig(32testreq)', buildreq.buildreqs) @@ -83,14 +79,14 @@ class TestBuildreq(unittest.TestCase): """ Test configure_ac_line with standard pattern """ - buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])') + buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False) self.assertIn('ncurses-devel', buildreq.buildreqs) def test_configure_ac_line_comment(self): """ Test configure_ac_line with commented line """ - buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])') + buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False) self.assertEqual(buildreq.buildreqs, set()) def test_configure_ac_line_pkg_check_modules(self): @@ -101,7 +97,7 @@ class TestBuildreq(unittest.TestCase): buildreq.configure_ac_line( 'PKG_CHECK_MODULES(prefix, ' '[module > 2 module2 < 2], ' - 'action-if-found, action-if-not-found)') + 'action-if-found, action-if-not-found)', False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) @@ -112,7 +108,7 @@ class TestBuildreq(unittest.TestCase): buildreq.configure_ac_line( 'XDT_CHECK_PACKAGE(prefix, ' '[module = 2 module2 > 9], ' - 'action-if-found, action-if-not-found)') + 'action-if-found, action-if-not-found)', False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) @@ -122,7 +118,7 @@ class TestBuildreq(unittest.TestCase): """ buildreq.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], ' 'action-if-found, ' - 'action-if-not-found)') + 'action-if-not-found)', False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(module1)', 'pkgconfig(module2)'])) @@ -143,7 +139,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_configure_ac(os.path.join(tmpd, 'fname')) + buildreq.parse_configure_ac(os.path.join(tmpd, 'fname'), False) self.assertEqual(buildreq.buildpattern.default_pattern, 'configure_ac') self.assertEqual(buildreq.buildreqs, @@ -205,7 +201,7 @@ class TestBuildreq(unittest.TestCase): content = 'does not matter, let us mock' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.parse_cargo_toml('filename') + buildreq.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3']) buildreq.os.path.exists = exists_backup buildreq.toml.loads = loads_backup @@ -249,11 +245,13 @@ class TestBuildreq(unittest.TestCase): """ Test rakefile parsing with both configured gems and unconfigured gems """ + conf = config.Config() + conf.setup_patterns() open_name = 'buildreq.util.open_auto' content = "line1\nrequire 'bundler/gem_tasks'\nline3\nrequire 'nope'" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.rakefile('filename') + buildreq.rakefile('filename', conf.gems) self.assertEqual(buildreq.buildreqs, set(['rubygem-rubygems-tasks'])) @@ -291,7 +289,7 @@ class TestBuildreq(unittest.TestCase): 'req7 == 3.3.3\n' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename') + buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) @@ -307,7 +305,7 @@ class TestBuildreq(unittest.TestCase): ' req7 == 3.3.3\n ' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename') + buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) @@ -321,7 +319,7 @@ class TestBuildreq(unittest.TestCase): "setup_requires=['req2']" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) self.assertEqual(buildreq.requires, set(['req1'])) @@ -336,7 +334,7 @@ class TestBuildreq(unittest.TestCase): "'req3']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) @@ -354,7 +352,7 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) @@ -372,7 +370,7 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) self.assertEqual(buildreq.requires, set(['req1', 'req2'])) @@ -385,7 +383,7 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=[reqname, 'req1', 'req2']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) self.assertEqual(buildreq.requires, set(['req1', 'req2'])) @@ -398,7 +396,7 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=reqname" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', []) self.assertEqual(buildreq.buildreqs, set()) self.assertEqual(buildreq.requires, set()) @@ -425,6 +423,7 @@ class TestBuildreq(unittest.TestCase): much to test here that uses the same logic, a representative test should be sufficient. """ + conf = config.Config() with tempfile.TemporaryDirectory() as tmpd: os.mkdir(os.path.join(tmpd, 'subdir')) open(os.path.join(tmpd, 'subdir', 'test.go'), 'w').close() @@ -433,7 +432,7 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'SConstruct'), 'w').close() open(os.path.join(tmpd, 'meson.build'), 'w').close() - buildreq.scan_for_configure(tmpd, "", "") + buildreq.scan_for_configure(tmpd, "", "", conf) self.assertEqual(buildreq.buildreqs, set(['buildreq-golang', 'buildreq-cmake', 'buildreq-scons', 'buildreq-distutils3', 'buildreq-meson'])) @@ -443,6 +442,7 @@ class TestBuildreq(unittest.TestCase): Test scan_for_configure when distutils is being used for the build pattern to test pypi metadata handling. """ + conf = config.Config() orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score orig_pypi_name = buildreq.pypidata.get_pypi_name @@ -459,7 +459,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: os.mkdir(os.path.join(tmpd, 'subdir')) open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd) + buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary @@ -476,6 +476,7 @@ class TestBuildreq(unittest.TestCase): Test scan_for_configure when distutils is being used for the build pattern to test pypi metadata file override handling. """ + conf = config.Config() open_name = 'buildreq.open' orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score @@ -491,7 +492,7 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() open(os.path.join(tmpd, 'pypi.json'), 'w').close() with patch(open_name, m_open, create=True): - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd) + buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary @@ -506,11 +507,13 @@ class TestBuildreq(unittest.TestCase): Test parse_cmake to ensure accurate detection of versioned and unversioned pkgconfig modules. """ + conf = config.Config() + conf.setup_patterns() content = 'pkg_check_modules(GLIB gio-unix-2.0>=2.46.0 glib-2.0 REQUIRED)' with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) @@ -520,11 +523,13 @@ class TestBuildreq(unittest.TestCase): Test parse_cmake to ensure accurate handling of versioned pkgconfig modules with whitespace. """ + conf = config.Config() + conf.setup_patterns() content = 'pkg_check_modules(GLIB gio-unix-2.0 >= 2.46.0 glib-2.0 REQUIRED)' with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) @@ -533,6 +538,8 @@ class TestBuildreq(unittest.TestCase): """ Test parse_cmake to ensure it ignores pkg_check_modules in comments. """ + conf = config.Config() + conf.setup_patterns() content = ''' # For example, consider the following patch to some CMakeLists.txt. # - pkg_check_modules(FOO REQUIRED foo>=1.0) @@ -541,7 +548,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set([])) @@ -551,11 +558,13 @@ class TestBuildreq(unittest.TestCase): Test parse_cmake to ensure accurate handling of versioned pkgconfig modules with variable version strings. """ + conf = config.Config() + conf.setup_patterns() content = 'pkg_check_modules(AVCODEC libavcodec${_avcodec_ver} libavutil$_avutil_ver)' with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)'])) @@ -564,7 +573,7 @@ class TestBuildreq(unittest.TestCase): """ Test parse_cmake to ensure accurate handling of find_package. """ - buildreq.config.cmake_modules = { + cmake_modules = { "valid": "valid", "valid_but_commented": "valid_but_commented", "different_name": "another_name", @@ -578,7 +587,7 @@ find_package(different_name) with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['valid', 'another_name'])) @@ -679,44 +688,37 @@ find_package(different_name) result = buildreq._get_desc_field("Field2", "\n".join(lines)) self.assertEqual(result, []) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_depends(self, os_pkgs): + def test_parse_r_desc_depends(self): """Test parsing of R description Depends field.""" pkgs = ['R-pkg1'] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = 'Depends: pkg1' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg1' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_imports(self, os_pkgs): + def test_parse_r_desc_imports(self): """Test parsing of an R description Imports field.""" pkgs = ['R-pkg2'] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = 'Imports: pkg2' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg2' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_linkingto(self, os_pkgs): + def test_parse_r_desc_linkingto(self): """Test parsing of an R description LinkingTo field.""" pkgs = ['R-pkg3'] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = 'LinkingTo: pkg3' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg3' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_multiple(self, os_pkgs): + def test_parse_r_desc_multiple(self): """Test parsing of an R description file that captures multiple fields.""" pkgs = [ 'R-pkg1', @@ -724,7 +726,6 @@ find_package(different_name) 'R-pkg3', 'R-pkg4', ] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = [ 'Field1: foo', @@ -735,7 +736,7 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertFalse('R-foo' in buildreq.buildreqs) self.assertFalse('R-bar' in buildreq.buildreqs) self.assertTrue('R-pkg1' in buildreq.buildreqs) @@ -743,13 +744,11 @@ find_package(different_name) self.assertTrue('R-pkg3' in buildreq.buildreqs) self.assertTrue('R-pkg4' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_not_in_os(self, os_pkgs): + def test_parse_r_desc_not_in_os(self): """Test parsing of an R description file with some non-OS packages.""" pkgs = [ 'R-pkg1', ] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = [ 'Imports: pkg1, pkg2', @@ -757,7 +756,7 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg1' in buildreq.buildreqs) self.assertFalse('R-pkg2' in buildreq.buildreqs) self.assertFalse('R-pkg3' in buildreq.buildreqs) diff --git a/tests/test_check.py b/tests/test_check.py index aaebfec..f8925f6 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import mock_open, patch import check - +import config def mock_generator(rv=None): def mock_f(*args, **kwargs): @@ -19,12 +19,6 @@ class TestTest(unittest.TestCase): @classmethod def setUpClass(self): self.open_name = 'check.util.open_auto' - check.config.config_opts['skip_tests'] = False - check.config.config_opts['allow_test_failures'] = False - check.config.config_opts['32bit'] = False - check.config.config_opts['use_avx2'] = False - check.config.config_opts['use_avx512'] = False - check.config.config_opts['openmpi'] = False check.os.path.isfile = mock_generator(True) @classmethod @@ -49,7 +43,7 @@ class TestTest(unittest.TestCase): m_open = mock_open() open_name = 'util.open' with patch(open_name, m_open, create=True): - check.check_regression('pkgdir') + check.check_regression('pkgdir', False) check.count.parse_log = parse_log_backup @@ -72,7 +66,7 @@ class TestTest(unittest.TestCase): m_open = mock_open() open_name = 'util.open' with patch(open_name, m_open, create=True): - check.check_regression('pkgdir') + check.check_regression('pkgdir', False) check.count.parse_log = parse_log_backup @@ -94,13 +88,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) content = 'check:' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -111,12 +106,13 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite via Makefile.am """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.am']) m_open = mock_open() with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure_ac" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -127,10 +123,11 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.PL']) check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test') @@ -139,13 +136,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite via Makefile.in """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) content = 'test:' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -155,13 +153,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with setup.py suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['setup.py']) content = 'test_suite' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "distutils3" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -173,13 +172,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with cmake suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['CMakeLists.txt']) content = 'enable_testing' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cmake" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -191,9 +191,10 @@ class TestTest(unittest.TestCase): Test scan_for_tests with tox.ini in the files list, should add several build requirements """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['tox.ini']) - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup self.assertEqual(check.buildreq.buildreqs, set(['tox', diff --git a/tests/test_commitmessage.py b/tests/test_commitmessage.py index b4b3b96..4256189 100644 --- a/tests/test_commitmessage.py +++ b/tests/test_commitmessage.py @@ -3,6 +3,7 @@ import unittest.mock as mock import os import tempfile import commitmessage +import config class TestCommitmessage(unittest.TestCase): @@ -10,7 +11,6 @@ class TestCommitmessage(unittest.TestCase): def setUp(self): commitmessage.tarball.name = 'testball' commitmessage.tarball.version = '0.0.1' - commitmessage.config.old_version = '0.0.0' self.workingdir = tempfile.TemporaryDirectory() commitmessage.build.setup_workingdir(self.workingdir.name) @@ -59,7 +59,7 @@ class TestCommitmessage(unittest.TestCase): # last items expected_msg = [""] + GOOD_NEWS.split('\n')[3:13] expected_cvs = set() - self.assertEqual(commitmessage.process_NEWS('NEWS'), + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), (expected_msg, expected_cvs)) def test_process_NEWS_bad_news(self): @@ -71,7 +71,7 @@ class TestCommitmessage(unittest.TestCase): with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile: # make GOOD_NEWS irrelevant by replacing current version newsfile.write(GOOD_NEWS.replace('0.0.1', '0.0.0')) - self.assertEqual(commitmessage.process_NEWS('NEWS'), ([], set())) + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), ([], set())) def test_process_NEWS_good_cves(self): """ @@ -90,7 +90,7 @@ class TestCommitmessage(unittest.TestCase): .replace('change2.2', 'CVE-2-2')\ .split('\n')[3:13] expected_cvs = set(['CVE-2-1', 'CVE-2-2']) - self.assertEqual(commitmessage.process_NEWS('NEWS'), + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), (expected_msg, expected_cvs)) def test_process_NEWS_long(self): @@ -111,7 +111,7 @@ class TestCommitmessage(unittest.TestCase): expected_msg.extend(['1', '2', '3', '4', '5', '6', '7', '', '(NEWS truncated at 15 lines)', '']) expected_cvs = set() - self.assertEqual(commitmessage.process_NEWS('NEWS'), + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), (expected_msg, expected_cvs)) def test_guess_commit_message(self): @@ -119,9 +119,11 @@ class TestCommitmessage(unittest.TestCase): Test guess_commit_message() with mocked internal functions and both commitmessage information and cves available from newsfile. """ + conf = config.Config() + conf.old_version = "0.0.0" process_NEWS_backup = commitmessage.process_NEWS - def mock_process_NEWS(newsfile): + def mock_process_NEWS(newsfile, old_version): return (['', 'commit', 'message', 'with', 'cves', ''], set(['cve1', 'cve2'])) @@ -129,7 +131,7 @@ class TestCommitmessage(unittest.TestCase): open_name = 'util.open_auto' with mock.patch(open_name, create=True) as mock_open: mock_open.return_value = mock.MagicMock() - commitmessage.guess_commit_message("") + commitmessage.guess_commit_message("", conf) # reset mocks before asserting so a failure doesn't cascade to # other tests commitmessage.process_NEWS = process_NEWS_backup @@ -147,23 +149,23 @@ class TestCommitmessage(unittest.TestCase): also available from config, which changes the first line of the commmit message. """ + conf = config.Config() process_NEWS_backup = commitmessage.process_NEWS - def mock_process_NEWS(newsfile): + def mock_process_NEWS(newsfile, old_version): return (['', 'commit', 'message', 'with', 'cves', ''], set(['cve1', 'cve2'])) commitmessage.process_NEWS = mock_process_NEWS - commitmessage.config.cves = set(['CVE-1234-5678']) - commitmessage.config.old_version = None # Allow cve title to be set + conf.cves = set(['CVE-1234-5678']) + conf.old_version = None # Allow cve title to be set open_name = 'util.open_auto' with mock.patch(open_name, create=True) as mock_open: mock_open.return_value = mock.MagicMock() - commitmessage.guess_commit_message("") + commitmessage.guess_commit_message("", conf) # reset mocks before asserting so a failure doesn't cascade to # other tests commitmessage.process_NEWS = process_NEWS_backup - commitmessage.config.cves = set() fh = mock_open.return_value.__enter__.return_value fh.write.assert_called_with( 'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n' @@ -178,23 +180,23 @@ class TestCommitmessage(unittest.TestCase): message. Additionally there is imported key info that will be displayed at the end of the message. """ + conf = config.Config() process_NEWS_backup = commitmessage.process_NEWS - def mock_process_NEWS(newsfile): + def mock_process_NEWS(newsfile, old_version): return (['', 'commit', 'message', 'with', 'cves', ''], set(['cve1', 'cve2'])) commitmessage.process_NEWS = mock_process_NEWS - commitmessage.config.cves = set(['CVE-1234-5678']) - commitmessage.config.old_version = None # Allow cve title to be set + conf.cves = set(['CVE-1234-5678']) + conf.old_version = None # Allow cve title to be set open_name = 'util.open_auto' with mock.patch(open_name, create=True) as mock_open: mock_open.return_value = mock.MagicMock() - commitmessage.guess_commit_message("keyinfo content") + commitmessage.guess_commit_message("keyinfo content", conf) # reset mocks before asserting so a failure doesn't cascade to # other tests commitmessage.process_NEWS = process_NEWS_backup - commitmessage.config.cves = set() fh = mock_open.return_value.__enter__.return_value fh.write.assert_called_with( 'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n' @@ -206,12 +208,13 @@ class TestCommitmessage(unittest.TestCase): """ Tests scan_for_changes using temporary directories """ + conf = config.Config() with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'changelog.txt'), 'w') as newsfile: newsfile.write('new changelog file') with tempfile.TemporaryDirectory() as tmpd1: - commitmessage.scan_for_changes(tmpd1, tmpd) + commitmessage.scan_for_changes(tmpd1, tmpd, conf.transforms) self.assertTrue(os.path.isfile(tmpd1 + '/ChangeLog')) diff --git a/tests/test_files.py b/tests/test_files.py index fa7b011..75bb115 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -20,7 +20,8 @@ def mock_return(retval): class TestFiles(unittest.TestCase): def setUp(self): - self.fm = FileManager() + conf = config.Config() + self.fm = FileManager(conf) def test_banned_path(self): """ @@ -66,14 +67,14 @@ class TestFiles(unittest.TestCase): """ Test compat_exclude with a file that shouldn't be excluded. """ - config.config_opts['compat'] = True + self.fm.config.config_opts['compat'] = True self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so.1')) def test_compat_exclude_exclude_file(self): """ Test compat_exclude with a file that should be excluded. """ - config.config_opts['compat'] = True + self.fm.config.config_opts['compat'] = True self.assertTrue(self.fm.compat_exclude('/usr/lib64/libfoo.so')) def test_compat_exclude_not_compat_mode(self): @@ -81,7 +82,7 @@ class TestFiles(unittest.TestCase): Test compat_exclude with a file that should be excluded but isn't because the package isn't being run in compat mode. """ - config.config_opts['compat'] = False + self.fm.config.config_opts['compat'] = False self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so')) def test_file_pat_match(self): diff --git a/tests/test_infile_update_spec.py b/tests/test_infile_update_spec.py index 72cf27d..b10d2a2 100644 --- a/tests/test_infile_update_spec.py +++ b/tests/test_infile_update_spec.py @@ -1,5 +1,6 @@ import unittest +import config import infile_update_spec import specfiles @@ -8,7 +9,7 @@ class TestUpdateSpecfile(unittest.TestCase): def setUp(self): # url, version, name, release url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1') + self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config()) self.bb_dict = { "DEPENDS": "ncurses gettext-native", diff --git a/tests/test_license.py b/tests/test_license.py index b7ba083..9fcf424 100644 --- a/tests/test_license.py +++ b/tests/test_license.py @@ -7,6 +7,7 @@ from unittest.mock import patch, mock_open, MagicMock import pycurl +import config import download import license import util @@ -14,20 +15,17 @@ import util class TestLicense(unittest.TestCase): - @classmethod - def setUpClass(self): - license.config.setup_patterns() - def setUp(self): license.licenses = [] - license.config.license_fetch = None def test_add_license(self): """ Test add_license from valid string, Apache-2 should be translated to Apache-2.0 """ - self.assertTrue(license.add_license('Apache-2')) + conf = config.Config() + conf.setup_patterns() + self.assertTrue(license.add_license('Apache-2', conf.license_translations, conf.license_blacklist)) self.assertIn('Apache-2.0', license.licenses) def test_add_license_present(self): @@ -36,8 +34,10 @@ class TestLicense(unittest.TestCase): the licenses list. Should return True and should not modify the licenses list. GPL-3 translates to GPL-3.0. """ + conf = config.Config() + conf.setup_patterns() license.licenses.append('GPL-3.0') - self.assertTrue(license.add_license('GPL-3')) + self.assertTrue(license.add_license('GPL-3', conf.license_translations, conf.license_blacklist)) self.assertEqual(['GPL-3.0'], license.licenses) def test_add_license_blacklisted(self): @@ -45,18 +45,22 @@ class TestLicense(unittest.TestCase): Test add_license from string in license_blacklist. Should return False and should not modify the licenses list. """ + conf = config.Config() + conf.setup_patterns() # sanity check to make sure the licenses list is empty before the later # assertIn() call self.assertEqual(license.licenses, []) - self.assertFalse(license.add_license('License')) + self.assertFalse(license.add_license('License', conf.license_translations, conf.license_blacklist)) self.assertNotIn('License', license.licenses) def test_license_from_copying_hash(self): """ Test license_from_copying_hash with valid license file """ - license.license_from_copying_hash('tests/COPYING_TEST', '') + conf = config.Config() + conf.setup_patterns() + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) self.assertIn('GPL-3.0', license.licenses) def test_license_from_copying_hash_no_license_show(self): @@ -64,26 +68,26 @@ class TestLicense(unittest.TestCase): Test license_from_copying_hash with invalid hash and no license_show set """ - bkup_hash = license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')] + conf = config.Config() + conf.setup_patterns() # remove the hash from license_hashes - del(license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')]) - license.config.license_show = "license.show.url" - license.license_from_copying_hash('tests/COPYING_TEST', '') + del(conf.license_hashes[license.get_sha1sum('tests/COPYING_TEST')]) + conf.license_show = "license.show.url" + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) - # restore the hash - license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')] = bkup_hash self.assertEquals(license.licenses, []) def test_license_from_copying_hash_bad_license(self): """ Test license_from_copying_hash with invalid license file """ + conf = config.Config() content = util.get_contents("tests/COPYING_TEST").replace(b"GNU", b"SNU") m_open = MagicMock() m_open.__str__.return_value = content with patch('license.get_contents', m_open, create=True): - license.license_from_copying_hash('copying.txt', '') + license.license_from_copying_hash('copying.txt', '', conf) self.assertEquals(license.licenses, []) @@ -114,13 +118,14 @@ class TestLicense(unittest.TestCase): # set the mock curl download.pycurl.Curl = MockCurl - license.config.license_fetch = 'license.server.url' + conf = config.Config() + conf.license_fetch = 'license.server.url' # let's check that the proper thing is being printed as well out = StringIO() with redirect_stdout(out): with self.assertRaises(SystemExit): - license.license_from_copying_hash('tests/COPYING_TEST', '') + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) self.assertIn('Unable to fetch license.server.url: Test Exception', out.getvalue()) @@ -167,12 +172,13 @@ class TestLicense(unittest.TestCase): # set the mock curl download.pycurl.Curl = MockCurl - license.config.license_fetch = 'license.server.url' + conf = config.Config() + conf.license_fetch = 'license.server.url' # let's check that the proper thing is being printed as well out = StringIO() with redirect_stdout(out): - license.license_from_copying_hash('tests/COPYING_TEST', '') + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) self.assertIn('GPL-3.0', license.licenses) self.assertIn('License : GPL-3.0 (server)', out.getvalue()) @@ -187,6 +193,8 @@ class TestLicense(unittest.TestCase): """ Test scan_for_licenses in temporary directory with valid license file """ + conf = config.Config() + conf.setup_patterns() with open('tests/COPYING_TEST', 'rb') as copyingf: content = copyingf.read() @@ -198,7 +206,7 @@ class TestLicense(unittest.TestCase): for testf in ['testlib.c', 'testmain.c', 'testheader.h']: with open(os.path.join(tmpd, testf), 'w') as newtestf: newtestf.write('test content') - license.scan_for_licenses(tmpd) + license.scan_for_licenses(tmpd, conf) self.assertIn('GPL-3.0', license.licenses) @@ -208,6 +216,8 @@ class TestLicense(unittest.TestCase): Should not add any licenses, should print a fatal message, should exit with a status code of 1. """ + conf = config.Config() + conf.setup_patterns() with tempfile.TemporaryDirectory() as tmpd: # create some cruft for testing for testf in ['testlib.c', 'testmain.c', 'testheader.h']: @@ -217,7 +227,7 @@ class TestLicense(unittest.TestCase): out = StringIO() with redirect_stdout(out): with self.assertRaises(SystemExit) as thread: - license.scan_for_licenses(tmpd) + license.scan_for_licenses(tmpd, conf) self.assertEqual(thread.exception.code, 1) self.assertIn("Cannot find any license", out.getvalue()) diff --git a/tests/test_pkg_integrity.py b/tests/test_pkg_integrity.py index cc4a1c2..f42dd53 100644 --- a/tests/test_pkg_integrity.py +++ b/tests/test_pkg_integrity.py @@ -6,6 +6,7 @@ import tempfile import unittest from unittest.mock import patch +import config import download import pkg_integrity @@ -46,27 +47,30 @@ def mock_download_do_curl(url, dst=None): @patch('download.do_curl', mock_download_do_curl) class TestCheckFn(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False + def _mock_rewrite(self, path): + del path def test_check_matching_sign_url(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: pkey = "023A4420C7EC6914.pkey" shutil.copy(os.path.join(TESTKEYDIR, pkey), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd) - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertTrue(result) def test_check_with_existing_sign(self): """ Download signature for local verification """ + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd) - result = pkg_integrity.check(NOSIGN_PKT_URL, tmpd) + result = pkg_integrity.check(NOSIGN_PKT_URL, tmpd, conf) self.assertTrue(result) @@ -130,11 +134,8 @@ class TestDomainBasedVerifiers(unittest.TestCase): @patch('download.do_curl', mock_download_do_curl) class TestGEMShaVerifier(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False + def _mock_rewrite(self, path): + del path def _mock_get_gem_info(pkg): info = ''' @@ -153,67 +154,85 @@ class TestGEMShaVerifier(unittest.TestCase): @patch('pkg_integrity.GEMShaVerifier.get_rubygems_info', _mock_get_gem_info) def test_from_url(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: filen = os.path.basename(GEM_PKT) shutil.copy(os.path.join(TESTDIR, filen), tmpd) - result = pkg_integrity.check(GEM_PKT, tmpd) + result = pkg_integrity.check(GEM_PKT, tmpd, conf) self.assertTrue(result) @patch('pkg_integrity.GEMShaVerifier.get_rubygems_info', _mock_get_gem_info) def test_non_matchingsha(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: out_file = os.path.join(tmpd, os.path.basename(GEM_PKT)) f = open(out_file, 'wb') f.write(b'this is made up data that will force a failure') f.close() with self.assertRaises(SystemExit) as a: - pkg_integrity.check(GEM_PKT, tmpd) + pkg_integrity.check(GEM_PKT, tmpd, conf) self.assertEqual(a.exception.code, 1) @patch('download.do_curl', mock_download_do_curl) class TestGPGVerifier(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False + def _mock_rewrite(self, path): + del path def test_from_url(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertTrue(result) def test_invalid_key(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd) with open(os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD) + ".asc"), 'w') as ofile: ofile.write("Invalid signature") - result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd) + result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd, conf) self.assertIsNone(result) def test_key_not_found(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd) - result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd) + result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd, conf) self.assertIsNone(result) def test_from_disk(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL)) out_key = out_file + ".asc" shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd) - result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key) + result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key, conf) self.assertTrue(result) def test_non_matchingsig(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL)) @@ -221,19 +240,26 @@ class TestGPGVerifier(unittest.TestCase): f.write(b'made up date that will fail check') f.close() with self.assertRaises(SystemExit) as a: - pkg_integrity.check(PACKAGE_URL, tmpd) + pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertEqual(a.exception.code, 1) def test_result_on_non_existent_pkg_path(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz', 'NonExistentPKG.tar.gz', - 'NonExistentKey.asc') + 'NonExistentKey.asc', + conf) self.assertIsNone(result) def test_result_on_nosign_package(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd) - result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd) + result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd, conf) self.assertIsNone(result) @patch.object(pkg_integrity.GPGCli, 'exec_cmd') @@ -270,10 +296,13 @@ class TestGPGVerifier(unittest.TestCase): mock_exec.return_value = (b'', b'', 0) + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertTrue(result) self.assertEqual(mock_parse.call_count, 4) self.assertEqual(mock_exec.call_count, 3) @@ -308,11 +337,14 @@ class TestGPGVerifier(unittest.TestCase): mock_exec.return_value = (b'', b'', 0) + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) with self.assertRaises(SystemExit) as msg: - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertEqual(msg.exception.code, 1) self.assertEqual(mock_parse.call_count, 4) self.assertEqual(mock_exec.call_count, 2) @@ -335,12 +367,6 @@ class TestInputGetter(unittest.TestCase): class TestUtils(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False - def test_get_verifier(self): x = pkg_integrity.get_verifier('file.abcd') self.assertEqual(x, None) diff --git a/tests/test_specdescription.py b/tests/test_specdescription.py index 76fab91..80d882b 100644 --- a/tests/test_specdescription.py +++ b/tests/test_specdescription.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import mock_open, patch +import config import specdescription @@ -15,7 +16,6 @@ class TestSpecdescription(unittest.TestCase): specdescription.default_summary = "No detailed summary available" specdescription.default_summary_score = 0 specdescription.license.licenses = [] - specdescription.config.license_translations = {} def test_clean_license_string(self): """ @@ -48,7 +48,7 @@ class TestSpecdescription(unittest.TestCase): "%donewithdesc\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_spec("filename") + specdescription.description_from_spec("filename", {}, []) self.assertEqual(specdescription.default_description, "Here is the description section\n" @@ -68,7 +68,7 @@ class TestSpecdescription(unittest.TestCase): content = "# this is a specfile without much in it\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_spec("filename") + specdescription.description_from_spec("filename", {}, []) self.assertEqual(specdescription.default_description, "No detailed description available") @@ -94,7 +94,7 @@ class TestSpecdescription(unittest.TestCase): "donewithdesc:\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_pkginfo("filename") + specdescription.description_from_pkginfo("filename", {}, []) self.assertEqual(specdescription.default_description, "Here is the description section\n" @@ -113,7 +113,7 @@ class TestSpecdescription(unittest.TestCase): content = "# this is a pkginfo file without much in it\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_pkginfo("filename") + specdescription.description_from_pkginfo("filename", {}, []) self.assertEqual(specdescription.default_description, "No detailed description available") diff --git a/tests/test_specfile.py b/tests/test_specfile.py index 2df2268..353d836 100644 --- a/tests/test_specfile.py +++ b/tests/test_specfile.py @@ -1,5 +1,6 @@ import unittest import unittest.mock +import config import specfiles @@ -11,9 +12,10 @@ class TestSpecfileWrite(unittest.TestCase): self.WRITES = [] def setUp(self): + conf = config.Config() + conf.config_opts['dev_requires_extras'] = False url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2') - specfiles.config.config_opts['dev_requires_extras'] = False + self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf) def mock_write(string): self.WRITES.append(string) @@ -286,8 +288,6 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_scriplets base test """ - backup_read_conf_file = specfiles.config.read_conf_file - def mock_read_conf_file(name): prefix = "pre" if "pre" in name else "post" return ["{}-script line 1\n".format(prefix), @@ -299,12 +299,11 @@ class TestSpecfileWrite(unittest.TestCase): def writelines(stringlist): self.WRITES.extend(stringlist) - specfiles.config.read_conf_file = mock_read_conf_file + self.specfile.config.read_conf_file = mock_read_conf_file self.specfile.specfile = MockSpecfile() self.specfile.packages["ruby"] = ["rubyfile1"] self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"] self.specfile.write_scriplets() - specfiles.config.read_conf_file = backup_read_conf_file expect = ["\n%post ruby\n", "post-script line 1\n\n", "post-script line 2\n\n", @@ -320,8 +319,6 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_scriplets with only pre-scripts present. """ - backup_read_conf_file = specfiles.config.read_conf_file - def mock_read_conf_file(name): if "pre" in name: return ["pre-script line 1\n", @@ -335,12 +332,11 @@ class TestSpecfileWrite(unittest.TestCase): def writelines(stringlist): self.WRITES.extend(stringlist) - specfiles.config.read_conf_file = mock_read_conf_file + self.specfile.config.read_conf_file = mock_read_conf_file self.specfile.specfile = MockSpecfile() self.specfile.packages["ruby"] = ["rubyfile1"] self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"] self.specfile.write_scriplets() - specfiles.config.read_conf_file = backup_read_conf_file expect = ["\n%pre ruby\n", "pre-script line 1\n\n", "pre-script line 2\n\n", @@ -352,8 +348,6 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_scriplets with no scripts present. """ - backup_read_conf_file = specfiles.config.read_conf_file - def mock_read_conf_file(name): return [] @@ -362,12 +356,11 @@ class TestSpecfileWrite(unittest.TestCase): def writelines(stringlist): self.WRITES.extend(stringlist) - specfiles.config.read_conf_file = mock_read_conf_file + self.specfile.config.read_conf_file = mock_read_conf_file self.specfile.specfile = MockSpecfile() self.specfile.packages["ruby"] = ["rubyfile1"] self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"] self.specfile.write_scriplets() - specfiles.config.read_conf_file = backup_read_conf_file self.assertEqual([], self.WRITES) def test_write_files_base(self): diff --git a/tests/test_tarball.py b/tests/test_tarball.py index 6ed8ca3..4f31151 100644 --- a/tests/test_tarball.py +++ b/tests/test_tarball.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import Mock, patch +import config import tarball @@ -123,11 +124,12 @@ def detect_build_test_generator(url, build_pattern): def name_and_version_test_generator(url, name, version): """Create test for tarball.name_and_version method.""" - @patch('tarball.config.parse_config_versions', Mock(return_value={})) def generator(self): """Test template.""" + conf = config.Config() + conf.parse_config_versions = Mock(return_value={}) tarball.url = url - n, _, v = tarball.name_and_version('', '', Mock()) + n, _, v = tarball.name_and_version('', '', Mock(), conf) self.assertEqual(name, n) self.assertEqual(version, v) if "github.com" in url: @@ -217,7 +219,6 @@ class TestTarball(unittest.TestCase): tarball.process_go_archives(go_archives) self.assertEqual(go_archives, go_archives_expected) - @patch('tarball.config', Mock()) def test_process_multiver_archives(self): """Test for tarball.process_multiver_archives method.""" # Set up input values @@ -236,8 +237,9 @@ class TestTarball(unittest.TestCase): ] # Set up a return value for parse_config_versions method attrs = {'parse_config_versions.return_value': config_versions} - tarball.config.configure_mock(**attrs) - tarball.process_multiver_archives(main_src, multiver_archives) + conf = Mock() + conf.configure_mock(**attrs) + tarball.process_multiver_archives(main_src, multiver_archives, conf) self.assertEqual(multiver_archives, expected_multiver_archives) @patch('tarball.Source.set_prefix', Mock())