diff --git a/autospec/autospec.py b/autospec/autospec.py index 02b6082..1c07bc7 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -85,11 +85,11 @@ def check_requirements(use_git): sys.exit(1) -def load_specfile(specfile): +def load_specfile(conf, specfile): """Gather all information from static analysis into Specfile instance.""" - config.load_specfile(specfile) + conf.load_specfile(specfile) tarball.load_specfile(specfile) - specdescription.load_specfile(specfile) + specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ) license.load_specfile(specfile) buildreq.load_specfile(specfile) buildpattern.load_specfile(specfile) @@ -125,10 +125,10 @@ def save_mock_logs(path, iteration): os.rename(src, dest) -def write_prep(workingdir): +def write_prep(conf, workingdir): """Write metadata to the local workingdir when --prep-only is used.""" - if config.urlban: - used_url = re.sub(config.urlban, "localhost", tarball.url) + if conf.urlban: + used_url = re.sub(conf.urlban, "localhost", tarball.url) else: used_url = tarball.url @@ -239,6 +239,7 @@ def main(): def package(args, url, name, archives, workingdir, infile_dict): """Entry point for building a package with autospec.""" + conf = config.Config() check_requirements(args.git) build.setup_workingdir(workingdir) @@ -246,9 +247,9 @@ def package(args, url, name, archives, workingdir, infile_dict): # First, download the tarball, extract it and then do a set # of static analysis on the content of the tarball. # - filemanager = files.FileManager() - tarball.process(url, name, args.version, args.target, archives, filemanager) - config.create_versions(build.download_path, tarball.multi_version) + filemanager = files.FileManager(conf) + tarball.process(url, name, args.version, args.target, archives, filemanager, conf) + conf.create_versions(build.download_path, tarball.multi_version) # Search up one level from here to capture multiple versions _dir = tarball.path @@ -262,36 +263,36 @@ def package(args, url, name, archives, workingdir, infile_dict): except Exception: pass # Start one directory higher so we scan *all* versions for licenses - license.scan_for_licenses(os.path.dirname(_dir)) + license.scan_for_licenses(os.path.dirname(_dir), conf) exit(0) - config.setup_patterns() - config.config_file = args.config - config.parse_config_files(build.download_path, args.bump, filemanager, tarball.version) - config.setup_patterns(config.failed_pattern_dir) - config.parse_existing_spec(build.download_path, tarball.name) + conf.setup_patterns() + conf.config_file = args.config + conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version) + conf.setup_patterns(conf.failed_pattern_dir) + conf.parse_existing_spec(build.download_path, tarball.name) if args.prep_only: - write_prep(workingdir) + write_prep(conf, workingdir) exit(0) buildreq.set_build_req() - buildreq.scan_for_configure(_dir, tarball.name, build.download_path) - specdescription.scan_for_description(tarball.name, _dir) + buildreq.scan_for_configure(_dir, tarball.name, build.download_path, conf) + specdescription.scan_for_description(tarball.name, _dir, conf.license_translations, conf.license_blacklist) # Start one directory higher so we scan *all* versions for licenses - license.scan_for_licenses(os.path.dirname(_dir)) - commitmessage.scan_for_changes(build.download_path, _dir) + license.scan_for_licenses(os.path.dirname(_dir), conf) + commitmessage.scan_for_changes(build.download_path, _dir, conf.transforms) add_sources(build.download_path, archives) - check.scan_for_tests(_dir) + check.scan_for_tests(_dir, conf) # # Now, we have enough to write out a specfile, and try to build it. # We will then analyze the build result and learn information until the # package builds # - specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release) + specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release, conf) filemanager.load_specfile(specfile) - load_specfile(specfile) + load_specfile(conf, specfile) # # If infile is passed, parse it and overwrite the specfile configurations @@ -303,12 +304,12 @@ def package(args, url, name, archives, workingdir, infile_dict): if args.integrity: interactive_mode = not args.non_interactive - pkg_integrity.check(url, build.download_path, interactive=interactive_mode) + pkg_integrity.check(url, build.download_path, conf, interactive=interactive_mode) pkg_integrity.load_specfile(specfile) specfile.write_spec(build.download_path) while 1: - build.package(filemanager, args.mock_config, args.mock_opts, args.cleanup) + build.package(filemanager, args.mock_config, args.mock_opts, conf, args.cleanup) filemanager.load_specfile(specfile) specfile.write_spec(build.download_path) filemanager.newfiles_printed = 0 @@ -326,10 +327,10 @@ def package(args, url, name, archives, workingdir, infile_dict): save_mock_logs(build.download_path, build.round) - check.check_regression(build.download_path) + check.check_regression(build.download_path, conf.config_opts['skip_tests']) if build.success == 0: - config.create_buildreq_cache(build.download_path, tarball.version) + conf.create_buildreq_cache(build.download_path, tarball.version) print_fatal("Build failed, aborting") sys.exit(1) elif os.path.isfile("README.clear"): @@ -345,18 +346,18 @@ def package(args, url, name, archives, workingdir, infile_dict): examine_abi(build.download_path) if os.path.exists("/var/lib/rpm"): - pkg_scan.get_whatrequires(tarball.name) + pkg_scan.get_whatrequires(tarball.name, conf.yum_conf) write_out(build.download_path + "/release", tarball.release + "\n") # record logcheck output logcheck(build.download_path) - commitmessage.guess_commit_message(pkg_integrity.IMPORTED) - config.create_buildreq_cache(build.download_path, tarball.version) + commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf) + conf.create_buildreq_cache(build.download_path, tarball.version) if args.git: - git.commit_to_git(build.download_path) + git.commit_to_git(build.download_path, conf) else: print("To commit your changes, git add the relevant files and " "run 'git commit -F commitmsg'") diff --git a/autospec/build.py b/autospec/build.py index c63a7f5..d577c87 100644 --- a/autospec/build.py +++ b/autospec/build.py @@ -24,7 +24,6 @@ import re import shutil import buildreq -import config import tarball import util @@ -45,13 +44,13 @@ def setup_workingdir(workingdir): download_path = os.path.join(base_path, tarball.name) -def simple_pattern_pkgconfig(line, pattern, pkgconfig): +def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32): """Check for pkgconfig patterns and restart build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: - must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, cache=True) + must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True) def simple_pattern(line, pattern, req): @@ -97,7 +96,7 @@ def cleanup_req(s: str) -> str: return s -def failed_pattern(line, pattern, verbose, buildtool=None): +def failed_pattern(line, config, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about @@ -119,11 +118,11 @@ def failed_pattern(line, pattern, verbose, buildtool=None): if req: must_restart += buildreq.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': - must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) + must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) elif buildtool == 'R': if buildreq.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 - buildreq.add_requires("R-" + s) + buildreq.add_requires("R-" + s, config.os_packages) elif buildtool == 'perl': s = s.replace('inc::', '') must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) @@ -167,7 +166,7 @@ def failed_pattern(line, pattern, verbose, buildtool=None): # Fallback to mvn-ARTIFACTID package name must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True) elif buildtool == 'catkin': - must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True) + must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) must_restart += buildreq.add_buildreq(s, cache=True) except Exception: @@ -207,7 +206,7 @@ def check_for_warning_pattern(line): util.print_warning("Build log contains: {}".format(pat)) -def parse_build_results(filename, returncode, filemanager): +def parse_build_results(filename, returncode, filemanager, config): """Handle build log contents.""" global must_restart global success @@ -222,13 +221,13 @@ def parse_build_results(filename, returncode, filemanager): for line in loglines: for pat in config.pkgconfig_pats: - simple_pattern_pkgconfig(line, *pat) + simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit')) for pat in config.simple_pats: simple_pattern(line, *pat) for pat in config.failed_pats: - failed_pattern(line, *pat) + failed_pattern(line, config, *pat) check_for_warning_pattern(line) @@ -277,7 +276,7 @@ def get_mock_cmd(): return 'sudo /usr/bin/mock' -def package(filemanager, mockconfig, mockopts, cleanup=False): +def package(filemanager, mockconfig, mockopts, config, cleanup=False): """Run main package build routine.""" global round global uniqueext @@ -343,7 +342,7 @@ def package(filemanager, mockconfig, mockopts, cleanup=False): is_clean = parse_buildroot_log(download_path + "/results/root.log", ret) if is_clean: - parse_build_results(download_path + "/results/build.log", ret, filemanager) + parse_build_results(download_path + "/results/build.log", ret, filemanager, config) if filemanager.has_banned: util.print_fatal("Content in banned paths found, aborting build") exit(1) diff --git a/autospec/buildreq.py b/autospec/buildreq.py index 59298b1..9e8f90b 100644 --- a/autospec/buildreq.py +++ b/autospec/buildreq.py @@ -23,10 +23,8 @@ import ast import json import os import re -import subprocess import buildpattern -import config import pypidata import specdescription import toml @@ -83,8 +81,8 @@ def add_buildreq(req, cache=False): return new -def add_requires(req, override=False): - """Add req to the global requires set if it is present in buildreqs and os_packages and is not banned.""" +def add_requires(req, packages, override=False): + """Add req to the global requires set if it is present in buildreqs and packages and is not banned.""" global buildreqs global requires new = True @@ -96,7 +94,7 @@ def add_requires(req, override=False): # Try dashes instead of underscores as some ecosystems are inconsistent in their naming req2 = req.replace("_", "-") - if req not in buildreqs and req2 in config.os_packages and req2 not in requires and req2 not in banned_requires: + if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: # Since this is done for python add a buildreq just in case (might not be correct though) buildreqs.add(req2) requires.add(req2) @@ -108,13 +106,13 @@ def add_requires(req, override=False): req2 = req[0].lower() + req[1:] else: req2 = req[0].upper() + req[1:] - if req not in buildreqs and req2 in config.os_packages and req2 not in requires and req2 not in banned_requires: + if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: # Since this is done for python add a buildreq just in case (might not be correct though) buildreqs.add(req2) requires.add(req2) return True - if req not in buildreqs and req not in config.os_packages and not override: + if req not in buildreqs and req not in packages and not override: if req: print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req)) return False @@ -124,16 +122,16 @@ def add_requires(req, override=False): return new -def add_pkgconfig_buildreq(preq, cache=False): +def add_pkgconfig_buildreq(preq, conf32, cache=False): """Format preq as pkgconfig req and add to buildreqs.""" - if config.config_opts['32bit']: + if conf32: req = "pkgconfig(32" + preq + ")" add_buildreq(req, cache) req = "pkgconfig(" + preq + ")" return add_buildreq(req, cache) -def configure_ac_line(line): +def configure_ac_line(line, conf32): """Parse configure_ac line and add appropriate buildreqs.""" # print("----\n", line, "\n----") # ignore comments @@ -164,7 +162,7 @@ def configure_ac_line(line): if len(L) > 1: rqlist = L[1].strip() for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req) + add_pkgconfig_buildreq(req, conf32) # PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found) match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line) @@ -172,7 +170,7 @@ def configure_ac_line(line): L = match.group(1).split(",") rqlist = L[0].strip() for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req) + add_pkgconfig_buildreq(req, conf32) def is_number(num_str): @@ -269,7 +267,7 @@ def parse_modules_list(modules_string, is_cmake=False): return res -def parse_configure_ac(filename): +def parse_configure_ac(filename, conf32): """Parse the configure.ac file for build requirements.""" buf = "" depth = 0 @@ -287,14 +285,14 @@ def parse_configure_ac(filename): if c != "\n": buf += c if c == "\n" and depth == 0: - configure_ac_line(buf) + configure_ac_line(buf, conf32) buf = "" - configure_ac_line(buf) + configure_ac_line(buf, conf32) f.close() -def parse_cargo_toml(filename): +def parse_cargo_toml(filename, packages): """Update build requirements using Cargo.toml. Set the build requirements for building rust programs using cargo. @@ -310,7 +308,7 @@ def parse_cargo_toml(filename): return for cdep in cargo["dependencies"]: if add_buildreq(cdep): - add_requires(cdep) + add_requires(cdep, packages) def _get_desc_field(field, desc): @@ -402,7 +400,7 @@ def _get_r_provides(): return set(provides) -def parse_r_description(filename): +def parse_r_description(filename, packages): """Update build/runtime requirements according to the R package description.""" deps = [] with util.open_auto(filename, "r") as desc: @@ -417,9 +415,9 @@ def parse_r_description(filename): if dep in r_provides: continue pkg = 'R-' + dep - if pkg in config.os_packages: + if pkg in packages: add_buildreq(pkg) - add_requires(pkg) + add_requires(pkg, packages) else: print("CRAN package '{}' not found in os_packages, skipping".format(pkg)) @@ -463,7 +461,7 @@ def set_build_req(): add_buildreq("rustc") -def rakefile(filename): +def rakefile(filename, gems): """Scan Rakefile for build requirements.""" with util.open_auto(filename, "r") as f: lines = f.readlines() @@ -473,14 +471,14 @@ def rakefile(filename): match = pat.search(line) if match: s = match.group(1) - if s != "rubygems" and s in config.gems: - print("Rakefile-dep: " + config.gems[s]) - add_buildreq(config.gems[s]) + if s != "rubygems" and s in gems: + print("Rakefile-dep: " + gems[s]) + add_buildreq(gems[s]) else: print("Rakefile-new: rubygem-" + s) -def parse_cmake(filename): +def parse_cmake(filename, cmake_modules, conf32): """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for.""" findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I) pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I) @@ -495,7 +493,7 @@ def parse_cmake(filename): if match: module = match.group(1) try: - pkg = config.cmake_modules[module] + pkg = cmake_modules[module] add_buildreq(pkg) except Exception: pass @@ -516,10 +514,10 @@ def parse_cmake(filename): module = wordmatch.group(2) # We have a match, so strip out any version info for m in parse_modules_list(module, is_cmake=True): - add_pkgconfig_buildreq(m) + add_pkgconfig_buildreq(m, conf32) -def qmake_profile(filename): +def qmake_profile(filename, qt_modules): """Scan .pro file for build requirements.""" with util.open_auto(filename, "r") as f: lines = f.readlines() @@ -533,7 +531,7 @@ def qmake_profile(filename): for module in s.split(): module = re.sub('-private$', '', module) try: - pc = config.qt_modules[module] + pc = qt_modules[module] add_buildreq('pkgconfig({})'.format(pc)) except Exception: pass @@ -575,7 +573,7 @@ def clean_python_req(req, add_python=True): return ret -def grab_python_requirements(descfile): +def grab_python_requirements(descfile, packages): """Add python requirements from requirements.txt file.""" if "/demo/" in descfile: return @@ -609,7 +607,7 @@ def grab_python_requirements(descfile): continue if clean_python_req(line) == 'mock': continue - add_requires(clean_python_req(line)) + add_requires(clean_python_req(line), packages) def get_python_build_version_from_classifier(filename): @@ -627,7 +625,7 @@ def get_python_build_version_from_classifier(filename): return "distutils3" -def add_setup_py_requires(filename): +def add_setup_py_requires(filename, packages): """Detect build requirements listed in setup.py in the install_requires and setup_requires lists. Handles the following patterns: @@ -675,7 +673,7 @@ def add_setup_py_requires(filename): dep = clean_python_req(ast.literal_eval(item), False) add_buildreq(dep) if req: - add_requires(dep) + add_requires(dep, packages) except Exception: # do not fail, the line contained a variable and @@ -699,7 +697,7 @@ def add_setup_py_requires(filename): dep = clean_python_req(ast.literal_eval(line), False) add_buildreq(dep) if req: - add_requires(dep) + add_requires(dep, packages) except Exception: # Do not fail, just keep looking @@ -721,7 +719,7 @@ def add_setup_py_requires(filename): dep = clean_python_req(dep) add_buildreq(dep) if req: - add_requires(dep) + add_requires(dep, packages) except Exception: # do not fail, the line contained a variable and had to @@ -729,7 +727,7 @@ def add_setup_py_requires(filename): pass -def parse_catkin_deps(cmakelists_file): +def parse_catkin_deps(cmakelists_file, conf32): """Determine requirements for catkin packages.""" f = util.open_auto(cmakelists_file, "r") lines = f.readlines() @@ -746,7 +744,7 @@ def parse_catkin_deps(cmakelists_file): comp = match.group("comp") if comp: for curr in comp.split(" "): - add_pkgconfig_buildreq(curr) + add_pkgconfig_buildreq(curr, conf32) catkin = True @@ -767,7 +765,7 @@ def is_qmake_pro(f): return f.endswith(".pro") and not f.startswith(".") -def scan_for_configure(dirn, tname, dlpath): +def scan_for_configure(dirn, tname, dlpath, config): """Scan the package directory for build files to determine build pattern.""" global pypi_provides global pypi_requires @@ -789,7 +787,7 @@ def scan_for_configure(dirn, tname, dlpath): add_buildreq("buildreq-scons") elif buildpattern.default_pattern == "R": add_buildreq("buildreq-R") - parse_r_description(os.path.join(dirn, "DESCRIPTION")) + parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages) elif buildpattern.default_pattern == "phpize": add_buildreq("buildreq-php") elif buildpattern.default_pattern == "nginx": @@ -819,7 +817,7 @@ def scan_for_configure(dirn, tname, dlpath): pkg = "go-" + req[0].replace("/", "-") add_buildreq(pkg) if buildpattern.default_pattern == "godep": - add_requires(pkg) + add_requires(pkg, config.os_packages) if "CMakeLists.txt" in files and "configure.ac" not in files: add_buildreq("buildreq-cmake") @@ -827,7 +825,7 @@ def scan_for_configure(dirn, tname, dlpath): srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or "..")) if os.path.samefile(dirpath, srcdir): - parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt")) + parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit')) if "configure" in files and os.access(dirpath + '/configure', os.X_OK): buildpattern.set_build_pattern("configure", default_score) @@ -836,11 +834,11 @@ def scan_for_configure(dirn, tname, dlpath): buildpattern.set_build_pattern("qmake", default_score) if "requires.txt" in files: - grab_python_requirements(dirpath + '/requires.txt') + grab_python_requirements(dirpath + '/requires.txt', config.os_packages) if "setup.py" in files: add_buildreq("buildreq-distutils3") - add_setup_py_requires(dirpath + '/setup.py') + add_setup_py_requires(dirpath + '/setup.py', config.os_packages) python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py') buildpattern.set_build_pattern(python_pattern, default_score) @@ -853,7 +851,7 @@ def scan_for_configure(dirn, tname, dlpath): buildpattern.set_build_pattern("scons", default_score) if "requirements.txt" in files: - grab_python_requirements(dirpath + '/requirements.txt') + grab_python_requirements(dirpath + '/requirements.txt', config.os_packages) if "meson.build" in files: add_buildreq("buildreq-meson") @@ -865,13 +863,13 @@ def scan_for_configure(dirn, tname, dlpath): for name in files: if name.lower() == "cargo.toml" and dirpath == dirn: - parse_cargo_toml(os.path.join(dirpath, name)) + parse_cargo_toml(os.path.join(dirpath, name), config.os_packages) if name.lower().startswith("configure."): - parse_configure_ac(os.path.join(dirpath, name)) + parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit')) if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby": - rakefile(os.path.join(dirpath, name)) + rakefile(os.path.join(dirpath, name), config.gems) if name.endswith(".pro") and buildpattern.default_pattern == "qmake": - qmake_profile(os.path.join(dirpath, name)) + qmake_profile(os.path.join(dirpath, name), config.qt_modules) if name.lower() == "makefile": buildpattern.set_build_pattern("make", default_score) if name.lower() == "autogen.sh": @@ -880,7 +878,7 @@ def scan_for_configure(dirn, tname, dlpath): buildpattern.set_build_pattern("cmake", default_score) if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \ and buildpattern.default_pattern == "cmake": - parse_cmake(os.path.join(dirpath, name)) + parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit')) can_reconf = os.path.exists(os.path.join(dirn, "configure.ac")) if not can_reconf: diff --git a/autospec/check.py b/autospec/check.py index a14edf6..a1d355f 100644 --- a/autospec/check.py +++ b/autospec/check.py @@ -24,7 +24,6 @@ import re import buildpattern import buildreq -import config import count import tarball import util @@ -32,9 +31,9 @@ import util tests_config = "" -def check_regression(pkg_dir): +def check_regression(pkg_dir, skip_tests): """Check the build log for test regressions using the count module.""" - if config.config_opts['skip_tests']: + if skip_tests: return result = count.parse_log(os.path.join(pkg_dir, "results/build.log")) @@ -56,11 +55,11 @@ def check_regression(pkg_dir): util.write_out(os.path.join(pkg_dir, "testresults"), res_str) -def scan_for_tests(src_dir): +def scan_for_tests(src_dir, config): """Scan source directory for test files and set tests_config accordingly.""" global tests_config - if config.config_opts['skip_tests'] or tests_config: + if config.config_opts.get('skip_tests') or tests_config: return makeflags = "%{?_smp_mflags} " if config.parallel_build else "" @@ -71,7 +70,7 @@ def scan_for_tests(src_dir): cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \ "make test\nmodule unload openmpi" - if config.config_opts['allow_test_failures']: + if config.config_opts.get('allow_test_failures'): make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \ "make VERBOSE=1 V=1 {}check || :\nmodule unload openmpi".format(makeflags) cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \ @@ -80,7 +79,7 @@ def scan_for_tests(src_dir): perl_check = "make TEST_VERBOSE=1 test" setup_check = """PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") python setup.py test""" meson_check = "meson test -C builddir" - if config.config_opts['allow_test_failures']: + if config.config_opts.get('allow_test_failures'): make_check += " || :" cmake_check += " || :" perl_check += " || :" @@ -96,17 +95,17 @@ def scan_for_tests(src_dir): "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd", "meson": meson_check, } - if config.config_opts['32bit']: + if config.config_opts.get('32bit'): testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :" testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :" - if config.config_opts['use_avx2']: + if config.config_opts.get('use_avx2'): testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :" - if config.config_opts['use_avx512']: + if config.config_opts.get('use_avx512'): testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :" testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :" - if config.config_opts['openmpi']: + if config.config_opts.get('openmpi'): testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi diff --git a/autospec/commitmessage.py b/autospec/commitmessage.py index 632c5dd..05cc74c 100644 --- a/autospec/commitmessage.py +++ b/autospec/commitmessage.py @@ -29,25 +29,24 @@ import sys from subprocess import PIPE, run import build -import config import tarball import util -def scan_for_changes(download_path, directory): +def scan_for_changes(download_path, directory, transforms): """Scan for changelogs or news files in the file sources. Scan for changelogs or news files in the source code and copy them to download_path as their - `config.transform`ed name. The file with the transformed name will later be parsed to find the + `transform`ed name. The file with the transformed name will later be parsed to find the commit message. """ found = [] - interests = config.transforms.keys() + interests = transforms.keys() for dirpath, dirnames, files in os.walk(directory, topdown=False): hits = [x for x in files if x.lower() in interests and x.lower() not in found] for item in hits: source = os.path.join(dirpath, item) - target = os.path.join(download_path, config.transforms[item.lower()]) + target = os.path.join(download_path, transforms[item.lower()]) try: shutil.copy(source, target) os.chmod(target, 0o644) @@ -82,7 +81,7 @@ def find_in_line(pattern, line): return bool(re.search(pattern, line)) -def process_NEWS(newsfile): +def process_NEWS(newsfile, old_version): """Parse the newfile for relevent changes. Look for changes and CVE fixes relevant to current version update. This information is returned @@ -99,7 +98,7 @@ def process_NEWS(newsfile): success = False start_found = False - if config.old_version is None or config.old_version == tarball.version: + if old_version is None or old_version == tarball.version: # no version update, so no information to search for in newsfile return commitmessage, cves @@ -113,7 +112,7 @@ def process_NEWS(newsfile): # escape some values for use in regular expressions below escaped_curver = re.escape(tarball.version) - escaped_oldver = re.escape(config.old_version) + escaped_oldver = re.escape(old_version) escaped_tarname = re.escape(tarball.name) # these are patterns that define the beginning of a block of information @@ -219,7 +218,7 @@ def process_git(giturl, oldversion, newversion): return shortlog -def guess_commit_message(keyinfo): +def guess_commit_message(keyinfo, config): """Parse newsfile for a commit message. Try and find a sane commit message for the newsfile. The commit message defaults to the @@ -271,7 +270,7 @@ def guess_commit_message(keyinfo): newsfiles = ["NEWS", "ChangeLog"] for newsfile in newsfiles: # parse news files for relevant version updates and cve fixes - newcommitmessage, newcves = process_NEWS(newsfile) + newcommitmessage, newcves = process_NEWS(newsfile, config.old_version) commitmessage.extend(newcommitmessage) cves.update(newcves) diff --git a/autospec/config.py b/autospec/config.py index b400248..292d6ea 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -35,349 +35,6 @@ import tarball from util import call, print_warning, write_out from util import open_auto -extra_configure = "" -extra_configure32 = "" -extra_configure64 = "" -extra_configure_avx2 = "" -extra_configure_avx512 = "" -config_files = set() -parallel_build = " %{?_smp_mflags} " -urlban = "" -extra_make = "" -extra32_make = "" -extra_make_install = "" -extra_make32_install = "" -extra_cmake = "" -extra_cmake_openmpi = "" -cmake_srcdir = "" -subdir = "" -install_macro = "%make_install" -disable_static = "--disable-static" -prep_prepend = [] -build_prepend = [] -build_append = [] -make_prepend = [] -install_prepend = [] -install_append = [] -service_restart = [] -patches = [] -verpatches = OrderedDict() -extra_sources = [] -autoreconf = False -custom_desc = "" -custom_summ = "" -set_gopath = True - -license_fetch = None -license_show = None -git_uri = None -os_packages = set() -config_file = None -old_version = None -old_patches = list() -old_keyid = None -profile_payload = None -signature = None -yum_conf = None -failed_pattern_dir = None -alias = None - -failed_commands = {} -ignored_commands = {} -maven_jars = {} -gems = {} -license_hashes = {} -license_translations = {} -license_blacklist = {} -qt_modules = {} -cmake_modules = {} - -cves = [] - -conf_args_openmpi = '--program-prefix= --exec-prefix=$MPI_ROOT \\\n' \ - '--libdir=$MPI_LIB --bindir=$MPI_BIN --sbindir=$MPI_BIN --includedir=$MPI_INCLUDE \\\n' \ - '--datarootdir=$MPI_ROOT/share --mandir=$MPI_MAN -exec-prefix=$MPI_ROOT --sysconfdir=$MPI_SYSCONFIG \\\n' \ - '--build=x86_64-generic-linux-gnu --host=x86_64-generic-linux-gnu --target=x86_64-clr-linux-gnu ' - -# Keep track of the package versions -versions = OrderedDict() -# Only parse the versions file once, and save the result for later -parsed_versions = OrderedDict() - -# defines which files to rename and copy to autospec directory, -# used in commitmessage.py -transforms = { - 'changes': 'ChangeLog', - 'changelog.txt': 'ChangeLog', - 'changelog': 'ChangeLog', - 'change.log': 'ChangeLog', - 'ChangeLog.md': 'ChangeLog', - 'changes.rst': 'ChangeLog', - 'changes.txt': 'ChangeLog', - 'news': 'NEWS', - 'meson_options.txt': 'meson_options.txt' -} - -config_opts = {} -config_options = { - "broken_c++": "extend flags with '-std=gnu++98", - "use_lto": "configure build for lto", - "use_avx2": "configure build for avx2", - "use_avx512": "configure build for avx512", - "keepstatic": "do not remove static libraries", - "asneeded": "unset %build LD_AS_NEEDED variable", - "allow_test_failures": "allow package to build with test failures", - "skip_tests": "Do not run test suite", - "no_autostart": "do not require autostart subpackage", - "optimize_size": "optimize build for size over speed", - "funroll-loops": "optimize build for speed over size", - "fast-math": "pass -ffast-math to compiler", - "insecure_build": "set flags to smallest -02 flags possible", - "conservative_flags": "set conservative build flags", - "broken_parallel_build": "disable parallelization during build", - "pgo": "set profile for pgo", - "use_clang": "add clang flags", - "32bit": "build 32 bit libraries", - "nostrip": "disable stripping binaries", - "verify_required": "require package verification for build", - "security_sensitive": "set flags for security-sensitive builds", - "so_to_lib": "add .so files to the lib package instead of dev", - "dev_requires_extras": "dev package requires the extras to be installed", - "autoupdate": "this package is trusted enough to automatically update " - "(used by other tools)", - "compat": "this package is a library compatibility package and only " - "ships versioned library files", - "nodebug": "do not generate debuginfo for this package", - "openmpi": "configure build also for openmpi"} - -# simple_pattern_pkgconfig patterns -# contains patterns for parsing build.log for missing dependencies -pkgconfig_pats = [ - (r"which: no qmake", "Qt"), - (r"XInput2 extension not found", "xi"), - (r"checking for UDEV\.\.\. no", "udev"), - (r"checking for UDEV\.\.\. no", "libudev"), - (r"XMLLINT not set and xmllint not found in path", "libxml-2.0"), - (r"error\: xml2-config not found", "libxml-2.0"), - (r"error: must install xorg-macros", "xorg-macros")] - -# simple_pattern patterns -# contains patterns for parsing build.log for missing dependencies -simple_pats = [ - (r'warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/.*"', "docbook-xml"), - (r"gobject-introspection dependency was not found, gir cannot be generated.", "gobject-introspection-dev"), - (r"gobject-introspection dependency was not found, gir cannot be generated.", "glibc-bin"), - (r"Cannot find development files for any supported version of libnl", "libnl-dev"), - (r"/", "cmake"), - (r"\-\- Boost libraries:", "boost-dev"), - (r"XInput2 extension not found", "inputproto"), - (r"^WARNING: could not find 'runtest'$", "dejagnu"), - (r"^WARNING: could not find 'runtest'$", "expect"), - (r"^WARNING: could not find 'runtest'$", "tcl"), - (r"VignetteBuilder package required for checking but installed:", "R-knitr"), - (r"You must have XML::Parser installed", "perl(XML::Parser)"), - (r"checking for Apache .* module support", "httpd-dev"), - (r"checking for.*in -ljpeg... no", "libjpeg-turbo-dev"), - (r"fatal error\: zlib\.h\: No such file or directory", "zlib-dev"), - (r"\* tclsh failed", "tcl"), - (r"\/usr\/include\/python3\.[0-9]+m\/pyconfig.h", "python3-dev"), - (r"checking \"location of ncurses\.h file\"", "ncurses-dev"), - (r"Can't exec \"aclocal\"", "automake"), - (r"Can't exec \"aclocal\"", "libtool"), - (r"configure: error: no suitable Python interpreter found", "python3-dev"), - (r"Checking for header Python.h", "python3-dev"), - (r"configure: error: No curses header-files found", "ncurses-dev"), - (r" \/usr\/include\/python3\.", "python3-dev"), - (r"to compile python extensions", "python3-dev"), - (r"testing autoconf... not found", "autoconf"), - (r"configure\: error\: could not find Python headers", "python3-dev"), - (r"checking for libxml libraries", "libxml2-dev"), - (r"checking for slang.h... no", "slang-dev"), - (r"configure: error: no suitable Python interpreter found", "python3"), - (r"configure: error: pcre-config for libpcre not found", "pcre"), - (r"checking for OpenSSL", "openssl-dev"), - (r"Package systemd was not found in the pkg-config search path.", "systemd-dev"), - (r"Unable to find the requested Boost libraries.", "boost-dev"), - (r"libproc not found. Please configure without procps", "procps-ng-dev"), - (r"configure: error: glib2", "glib-dev"), - (r"C library 'efivar' not found", "efivar-dev"), - (r"Has header \"efi.h\": NO", "gnu-efi-dev"), - (r"ERROR: Could not execute Vala compiler", "vala"), - (r".*: error: HAVE_INTROSPECTION does not appear in AM_CONDITIONAL", 'gobject-introspection-dev')] - -# failed_pattern patterns -# contains patterns for parsing build.log for missing dependencies -failed_pats = [ - (r" ! ([a-zA-Z:]+) is not installed", 0, 'perl'), - (r" ([a-zA-Z]+\:\:[a-zA-Z]+) not installed", 1, None), - (r"(?:Could|Did) (?:NOT|not) find ([a-zA-Z0-9]+)", 0, None), - (r" ([a-zA-Z0-9\-]*\.m4) not found", 0, None), - (r" exec: ([a-zA-Z0-9\-]+): not found", 0, None), - (r"([a-zA-Z0-9\-\_\.]*)\: command not found", 1, None), - (r"([a-zA-Z\-]*) (?:validation )?tool not found or not executable", 0, None), - (r"([a-zA-Z\-]+) [0-9\.]+ is required to configure this module; " - r"please install it or upgrade your CPAN\/CPANPLUS shell.", 0, None), - (r"-- (.*) not found.", 1, None), - (r".* /usr/bin/([a-zA-Z0-9-_]*).*not found", 0, None), - (r".*\.go:.*cannot find package \"(.*)\" in any of:", 0, 'go'), - (r"/usr/bin/env\: (.*)\: No such file or directory", 0, None), - (r"/usr/bin/python.*\: No module named (.*)", 0, None), - (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:\/]+)", 0, 'ruby table'), - (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:]+) ", 0, 'ruby'), - (r"Add the installation prefix of \"(.*)\" to CMAKE_PREFIX_PATH", 0, None), - (r"By not providing \"([a-zA-Z0-9]+).cmake\" in CMAKE_MODULE_PATH this project", 0, None), - (r"C library '(.*)' not found", 0, None), - (r"CMake Error at cmake\/modules\/([a-zA-Z0-9]+).cmake", 0, None), - (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC " r"\(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'), - (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None), - (r"Checking for (.*?)\.\.\.no", 0, None), - (r"Checking for (.*?)\s*: not found", 0, None), - (r"Checking for (.*?)\s>=.*\s*: not found", 0, None), - (r"Could not find '([a-zA-Z0-9\-\_]*)' \([~<>=]+ ([0-9.]+).*\) among [0-9]+ total gem", 0, 'ruby'), - (r"Could not find gem '([a-zA-Z0-9\-\_]+) \([~<>=0-9\.\, ]+\) ruby'", 0, 'ruby'), - (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None), - (r"Download error on https://pypi.python.org/simple/([a-zA-Z0-9\-\._:]+)/", 0, 'pypi'), - (r"Downloading https?://.*\.python\.org/packages/.*/.?/([A-Za-z]*)/.*", 0, None), - (r"ERROR: Could not find a valid gem '([a-zA-Z0-9\-\_\:]*)' \([>=]+ ([0-9.]+).*\)", 0, 'ruby'), - (r"ERROR: dependencies ['‘]([a-zA-Z0-9\-\.]*)['’].* are not available for package ['‘].*['’]", 0, 'R'), - (r"ERROR: dependencies ['‘].*['’], ['‘]([a-zA-Z0-9\-\.]*)['’],.* are not available for package ['‘].*['’]", 0, 'R'), - (r"ERROR: dependencies.*['‘]([a-zA-Z0-9\-\.]*)['’] are not available for package ['‘].*['’]", 0, 'R'), - (r"ERROR: dependency ['‘]([a-zA-Z0-9\-\.]*)['’] is not available for package ['‘].*['’]", 0, 'R'), - (r"Error: Unable to find (.*)", 0, None), - (r"Error: package ['‘]([a-zA-Z0-9\-\.]*)['’] required by", 0, 'R'), - (r"Gem::LoadError: Could not find '([a-zA-Z0-9\-\_]*)'", 0, 'ruby'), - (r"ImportError:.* No module named '?([a-zA-Z0-9\-\._]+)'?", 0, 'pypi'), - (r"ImportError\: ([a-zA-Z]+) module missing", 0, None), - (r"ImportError\: (?:No module|cannot import) named? (.*)", 0, None), - (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:\/\_]+)", 0, 'ruby table'), - (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:]+)/.*", 0, 'ruby'), - (r"ModuleNotFoundError.*No module named (.*)", 0, None), - (r"Native dependency '(.*)' not found", 0, "pkgconfig"), - (r"No library found for -l([a-zA-Z\-])", 0, None), - (r"No (?:matching distribution|local packages or working download links) found for ([a-zA-Z0-9\-\.\_]+)", 0, 'pypi'), - (r"No package '([a-zA-Z0-9\-:]*)' found", 0, 'pkgconfig'), - (r"No rule to make target `(.*)',", 0, None), - (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'), - (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'), - (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'), - (r"Program (.*) found: NO", 0, None), - (r"Target '[a-zA-Z0-9\-]' can't be generated as '(.*)' could not be found", 0, None), - (r"Unable to `import (.*)`", 0, None), - (r"Unable to find '(.*)'", 0, None), - (r"Unknown packages? ['‘]([a-zA-Z0-9\-]*)['’].* in Rd xrefs", 0, 'R'), - (r"WARNING: [a-zA-Z\-\_]+ dependency on ([a-zA-Z0-9\-\_:]*) \([<>=~]+ ([0-9.]+).*\) .*", 0, 'ruby'), - (r"Warning: prerequisite ([a-zA-Z:]+) [0-9\.]+ not found.", 0, 'perl'), - (r"Warning\: no usable ([a-zA-Z0-9]+) found", 0, None), - (r"You need ([a-zA-Z0-9\-\_]*) to build this program.", 1, None), - (r"[Dd]ependency (.*) found: NO \(tried pkgconfig(?: and cmake)?\)", 0, 'pkgconfig'), - (r"[Dd]ependency (.*) found: NO", 0, None), - (r"[a-zA-Z0-9\-:]* is not installed: cannot load such file -- rdoc/([a-zA-Z0-9\-:]*)", 0, 'ruby'), - (r"\-\- Could NOT find ([a-zA-Z0-9]+)", 0, None), - (r"\/bin\/ld: cannot find (-l[a-zA-Z0-9\_]+)", 0, None), - (r"^.*By not providing \"Find(.*).cmake\" in CMAKE_MODULE_PATH this.*$", 0, None), - (r"^.*Could not find a package configuration file provided by \"(.*)\".*$", 0, None), - (r"^.*\"(.*)\" with any of the following names.*$", 0, None), - (r"[Cc]hecking for (.*) (?:support|development files|with pkg-config)?\.\.\. [Nn]o", 0, None), - (r"checking (.*?)\.\.\. no", 0, None), - (r"checking for (.*) in default path\.\.\. not found", 0, None), - (r"checking for (.*)... configure: error", 0, None), - (r"checking for (.*?)\.\.\. no", 0, None), - (r"checking for [a-zA-Z0-9\_\-]+ in (.*?)\.\.\. no", 0, None), - (r"checking for library containing (.*)... no", 0, None), - (r"checking for perl module ([a-zA-Z:]+) [0-9\.]+... no", 0, 'perl'), - (r"configure: error: (?:pkg-config missing|Unable to locate) (.*)", 0, None), - (r"configure: error: ([a-zA-Z0-9]+) (?:is required to build|not found)", 0, None), - (r"configure: error: Cannot find (.*)\. Make sure", 0, None), - (r"fatal error\: (.*)\: No such file or directory", 0, None), - (r"make: ([a-zA-Z0-9].+): Command not found", 0, None), - (r"meson\.build\:[\d]+\:[\d]+\: ERROR: C library \'(.*)\' not found", 0, None), - (r"there is no package called ['‘]([a-zA-Z0-9\-\.]*)['’]", 0, 'R'), - (r"unable to execute '([a-zA-Z\-]*)': No such file or directory", 0, None), - (r"warning: failed to load external entity " r"\"(/usr/share/sgml/docbook/xsl-stylesheets)/.*\"", 0, None), - (r"which\: no ([a-zA-Z\-]*) in \(", 0, None), - (r"you may need to install the ([a-zA-Z0-9_\-:\.]*) module", 0, 'perl'), -] - - -def get_metadata_conf(): - """Gather package metadata from the tarball module.""" - metadata = {} - metadata['name'] = tarball.name - if urlban: - metadata['url'] = re.sub(urlban, "localhost", tarball.url) - metadata['archives'] = re.sub(urlban, "localhost", " ".join(tarball.archives)) - else: - metadata['url'] = tarball.url - metadata['archives'] = " ".join(tarball.archives) - - metadata['giturl'] = tarball.giturl - metadata['domain'] = tarball.domain - - if alias: - metadata['alias'] = alias - else: - metadata['alias'] = "" - - return metadata - - -def create_conf(path): - """Create options.conf file and use deprecated configuration files or defaults to populate.""" - config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) - - # first the metadata - config_f['package'] = get_metadata_conf() - - # next the options - config_f['autospec'] = {} - for fname, comment in sorted(config_options.items()): - config_f.set('autospec', '# {}'.format(comment)) - if os.path.exists(fname): - config_f['autospec'][fname] = 'true' - os.remove(fname) - else: - config_f['autospec'][fname] = 'false' - - # default lto to true for new things - config_f['autospec']['use_lto'] = 'true' - - # renamed options need special care - if os.path.exists("skip_test_suite"): - config_f['autospec']['skip_tests'] = 'true' - os.remove("skip_test_suite") - write_config(config_f, path) - - -def create_buildreq_cache(path, version): - """Make the buildreq_cache file.""" - content = read_conf_file(os.path.join(path, "buildreq_cache")) - # don't create an empty cache file - if len(buildreq.buildreqs_cache) < 1: - try: - # file was possibly added to git so we should clean it up - os.unlink(content) - except Exception: - pass - return - if not content: - pkgs = sorted(buildreq.buildreqs_cache) - else: - pkgs = sorted(set(content[1:]).union(buildreq.buildreqs_cache)) - with open(os.path.join(path, 'buildreq_cache'), "w") as cachefile: - cachefile.write("\n".join([version] + pkgs)) - config_files.add('buildreq_cache') - - -def create_versions(path, versions): - """Make versions file.""" - with open(os.path.join(path, "versions"), 'w') as vfile: - for version in versions: - vfile.write(version) - if versions[version]: - vfile.write('\t' + versions[version]) - vfile.write('\n') - config_files.add("versions") - def write_config(config_f, path): """Write the config_f to configfile.""" @@ -385,104 +42,6 @@ def write_config(config_f, path): config_f.write(configfile) -def read_config_opts(path): - """Read config options from path/options.conf.""" - global config_opts - global transforms - global alias - - opts_path = os.path.join(path, 'options.conf') - if not os.path.exists(opts_path): - create_conf(path) - - config_f = configparser.ConfigParser(interpolation=None) - config_f.read(opts_path) - if "autospec" not in config_f.sections(): - print("Missing autospec section in options.conf") - sys.exit(1) - - if 'package' in config_f.sections() and config_f['package'].get('alias'): - alias = config_f['package']['alias'] - - for key in config_f['autospec']: - config_opts[key] = config_f['autospec'].getboolean(key) - - # Rewrite the configuration file in case of formatting changes since a - # configuration file may exist without any comments (either due to an older - # version of autospec or if it was user-created) - rewrite_config_opts(path) - - # Don't use the ChangeLog files if the giturl is set - # ChangeLog is just extra noise when we can already see the gitlog - if "package" in config_f.sections() and config_f['package'].get('giturl'): - keys = [] - for k, v in transforms.items(): - if v == "ChangeLog": - keys.append(k) - for k in keys: - transforms.pop(k) - - -def rewrite_config_opts(path): - """Rewrite options.conf file when an option has changed (verify_required for example).""" - config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) - config_f['package'] = get_metadata_conf() - config_f['autospec'] = {} - - # Populate missing configuration options - # (in case of a user-created options.conf) - missing = set(config_options.keys()).difference(set(config_opts.keys())) - for option in missing: - config_opts[option] = False - - for fname, comment in sorted(config_options.items()): - config_f.set('autospec', '# {}'.format(comment)) - config_f['autospec'][fname] = 'true' if config_opts[fname] else 'false' - - write_config(config_f, path) - - -def read_file(path, track=True): - """Read full file at path. - - If the file does not exist (or is not expected to exist) - in the package git repo, specify 'track=False'. - """ - try: - with open(path, "r") as f: - if track: - config_files.add(os.path.basename(path)) - return f.readlines() - except EnvironmentError: - return [] - - -def read_conf_file(path, track=True): - """Read configuration file at path. - - If the config file does not exist (or is not expected to exist) - in the package git repo, specify 'track=False'. - """ - lines = read_file(path, track=track) - return [l.strip() for l in lines if not l.strip().startswith("#") and l.split()] - - -def read_script_file(path, track=True): - """Read RPM script snippet file at path. - - Returns verbatim, except for possibly the first line. - - If the config file does not exist (or is not expected to exist) - in the package git repo, specify 'track=False'. - """ - lines = read_file(path, track=track) - if len(lines) > 0 and (lines[0].startswith('#!') or lines[0].startswith('# -*- ')): - lines = lines[1:] - # Remove any trailing whitespace and newlines. The newlines are later - # restored by writer functions. - return [line.rstrip() for line in lines] - - def read_pattern_conf(filename, dest, list_format=False, path=None): """Read a fail-pattern configuration file. @@ -513,531 +72,888 @@ def read_pattern_conf(filename, dest, list_format=False, path=None): dest[pattern] = package.rstrip() -def setup_patterns(path=None): - """Read each pattern configuration file and assign to the appropriate variable.""" - global failed_commands - global ignored_commands - global maven_jars - global gems - global license_hashes - global license_translations - global license_blacklist - global qt_modules - global cmake_modules +class Config(object): + """Class to handle autospec configuration.""" - read_pattern_conf("ignored_commands", ignored_commands, list_format=True, path=path) - read_pattern_conf("failed_commands", failed_commands, path=path) - read_pattern_conf("maven_jars", maven_jars, path=path) - read_pattern_conf("gems", gems, path=path) - read_pattern_conf("license_hashes", license_hashes, path=path) - read_pattern_conf("license_translations", license_translations, path=path) - read_pattern_conf("license_blacklist", license_blacklist, list_format=True, path=path) - read_pattern_conf("qt_modules", qt_modules, path=path) - read_pattern_conf("cmake_modules", cmake_modules, path=path) + def __init__(self): + """Initialize Default configuration settings.""" + self.extra_configure = "" + self.extra_configure32 = "" + self.extra_configure64 = "" + self.extra_configure_avx2 = "" + self.extra_configure_avx512 = "" + self.config_files = set() + self.parallel_build = " %{?_smp_mflags} " + self.urlban = "" + self.extra_make = "" + self.extra32_make = "" + self.extra_make_install = "" + self.extra_make32_install = "" + self.extra_cmake = "" + self.extra_cmake_openmpi = "" + self.cmake_srcdir = "" + self.subdir = "" + self.install_macro = "%make_install" + self.disable_static = "--disable-static" + self.prep_prepend = [] + self.build_prepend = [] + self.build_append = [] + self.make_prepend = [] + self.install_prepend = [] + self.install_append = [] + self.service_restart = [] + self.patches = [] + self.verpatches = OrderedDict() + self.extra_sources = [] + self.autoreconf = False + self.custom_desc = "" + self.custom_summ = "" + self.set_gopath = True + self.license_fetch = None + self.license_show = None + self.git_uri = None + self.os_packages = set() + self.config_file = None + self.old_version = None + self.old_patches = list() + self.old_keyid = None + self.profile_payload = None + self.signature = None + self.yum_conf = None + self.failed_pattern_dir = None + self.alias = None + self.failed_commands = {} + self.ignored_commands = {} + self.maven_jars = {} + self.gems = {} + self.license_hashes = {} + self.license_translations = {} + self.license_blacklist = {} + self.qt_modules = {} + self.cmake_modules = {} + self.cves = [] + self.conf_args_openmpi = '--program-prefix= --exec-prefix=$MPI_ROOT \\\n' \ + '--libdir=$MPI_LIB --bindir=$MPI_BIN --sbindir=$MPI_BIN --includedir=$MPI_INCLUDE \\\n' \ + '--datarootdir=$MPI_ROOT/share --mandir=$MPI_MAN -exec-prefix=$MPI_ROOT --sysconfdir=$MPI_SYSCONFIG \\\n' \ + '--build=x86_64-generic-linux-gnu --host=x86_64-generic-linux-gnu --target=x86_64-clr-linux-gnu ' + # Keep track of the package versions + self.versions = OrderedDict() + # Only parse the versions file once, and save the result for later + self.parsed_versions = OrderedDict() + # defines which files to rename and copy to autospec directory, + # used in commitmessage.py + self.transforms = { + 'changes': 'ChangeLog', + 'changelog.txt': 'ChangeLog', + 'changelog': 'ChangeLog', + 'change.log': 'ChangeLog', + 'ChangeLog.md': 'ChangeLog', + 'changes.rst': 'ChangeLog', + 'changes.txt': 'ChangeLog', + 'news': 'NEWS', + 'meson_options.txt': 'meson_options.txt' + } + self.config_opts = {} + self.config_options = { + "broken_c++": "extend flags with '-std=gnu++98", + "use_lto": "configure build for lto", + "use_avx2": "configure build for avx2", + "use_avx512": "configure build for avx512", + "keepstatic": "do not remove static libraries", + "asneeded": "unset %build LD_AS_NEEDED variable", + "allow_test_failures": "allow package to build with test failures", + "skip_tests": "Do not run test suite", + "no_autostart": "do not require autostart subpackage", + "optimize_size": "optimize build for size over speed", + "funroll-loops": "optimize build for speed over size", + "fast-math": "pass -ffast-math to compiler", + "insecure_build": "set flags to smallest -02 flags possible", + "conservative_flags": "set conservative build flags", + "broken_parallel_build": "disable parallelization during build", + "pgo": "set profile for pgo", + "use_clang": "add clang flags", + "32bit": "build 32 bit libraries", + "nostrip": "disable stripping binaries", + "verify_required": "require package verification for build", + "security_sensitive": "set flags for security-sensitive builds", + "so_to_lib": "add .so files to the lib package instead of dev", + "dev_requires_extras": "dev package requires the extras to be installed", + "autoupdate": "this package is trusted enough to automatically update (used by other tools)", + "compat": "this package is a library compatibility package and only ships versioned library files", + "nodebug": "do not generate debuginfo for this package", + "openmpi": "configure build also for openmpi" + } + # simple_pattern_pkgconfig patterns + # contains patterns for parsing build.log for missing dependencies + self.pkgconfig_pats = [ + (r"which: no qmake", "Qt"), + (r"XInput2 extension not found", "xi"), + (r"checking for UDEV\.\.\. no", "udev"), + (r"checking for UDEV\.\.\. no", "libudev"), + (r"XMLLINT not set and xmllint not found in path", "libxml-2.0"), + (r"error\: xml2-config not found", "libxml-2.0"), + (r"error: must install xorg-macros", "xorg-macros") + ] + # simple_pattern patterns + # contains patterns for parsing build.log for missing dependencies + self.simple_pats = [ + (r'warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/.*"', "docbook-xml"), + (r"gobject-introspection dependency was not found, gir cannot be generated.", "gobject-introspection-dev"), + (r"gobject-introspection dependency was not found, gir cannot be generated.", "glibc-bin"), + (r"Cannot find development files for any supported version of libnl", "libnl-dev"), + (r"/", "cmake"), + (r"\-\- Boost libraries:", "boost-dev"), + (r"XInput2 extension not found", "inputproto"), + (r"^WARNING: could not find 'runtest'$", "dejagnu"), + (r"^WARNING: could not find 'runtest'$", "expect"), + (r"^WARNING: could not find 'runtest'$", "tcl"), + (r"VignetteBuilder package required for checking but installed:", "R-knitr"), + (r"You must have XML::Parser installed", "perl(XML::Parser)"), + (r"checking for Apache .* module support", "httpd-dev"), + (r"checking for.*in -ljpeg... no", "libjpeg-turbo-dev"), + (r"fatal error\: zlib\.h\: No such file or directory", "zlib-dev"), + (r"\* tclsh failed", "tcl"), + (r"\/usr\/include\/python3\.[0-9]+m\/pyconfig.h", "python3-dev"), + (r"checking \"location of ncurses\.h file\"", "ncurses-dev"), + (r"Can't exec \"aclocal\"", "automake"), + (r"Can't exec \"aclocal\"", "libtool"), + (r"configure: error: no suitable Python interpreter found", "python3-dev"), + (r"Checking for header Python.h", "python3-dev"), + (r"configure: error: No curses header-files found", "ncurses-dev"), + (r" \/usr\/include\/python3\.", "python3-dev"), + (r"to compile python extensions", "python3-dev"), + (r"testing autoconf... not found", "autoconf"), + (r"configure\: error\: could not find Python headers", "python3-dev"), + (r"checking for libxml libraries", "libxml2-dev"), + (r"checking for slang.h... no", "slang-dev"), + (r"configure: error: no suitable Python interpreter found", "python3"), + (r"configure: error: pcre-config for libpcre not found", "pcre"), + (r"checking for OpenSSL", "openssl-dev"), + (r"Package systemd was not found in the pkg-config search path.", "systemd-dev"), + (r"Unable to find the requested Boost libraries.", "boost-dev"), + (r"libproc not found. Please configure without procps", "procps-ng-dev"), + (r"configure: error: glib2", "glib-dev"), + (r"C library 'efivar' not found", "efivar-dev"), + (r"Has header \"efi.h\": NO", "gnu-efi-dev"), + (r"ERROR: Could not execute Vala compiler", "vala"), + (r".*: error: HAVE_INTROSPECTION does not appear in AM_CONDITIONAL", 'gobject-introspection-dev') + ] + # failed_pattern patterns + # contains patterns for parsing build.log for missing dependencies + self.failed_pats = [ + (r" ! ([a-zA-Z:]+) is not installed", 0, 'perl'), + (r" ([a-zA-Z]+\:\:[a-zA-Z]+) not installed", 1, None), + (r"(?:Could|Did) (?:NOT|not) find ([a-zA-Z0-9]+)", 0, None), + (r" ([a-zA-Z0-9\-]*\.m4) not found", 0, None), + (r" exec: ([a-zA-Z0-9\-]+): not found", 0, None), + (r"([a-zA-Z0-9\-\_\.]*)\: command not found", 1, None), + (r"([a-zA-Z\-]*) (?:validation )?tool not found or not executable", 0, None), + (r"([a-zA-Z\-]+) [0-9\.]+ is required to configure this module; " + r"please install it or upgrade your CPAN\/CPANPLUS shell.", 0, None), + (r"-- (.*) not found.", 1, None), + (r".* /usr/bin/([a-zA-Z0-9-_]*).*not found", 0, None), + (r".*\.go:.*cannot find package \"(.*)\" in any of:", 0, 'go'), + (r"/usr/bin/env\: (.*)\: No such file or directory", 0, None), + (r"/usr/bin/python.*\: No module named (.*)", 0, None), + (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:\/]+)", 0, 'ruby table'), + (r":in `require': cannot load such file -- ([a-zA-Z0-9\-\_:]+) ", 0, 'ruby'), + (r"Add the installation prefix of \"(.*)\" to CMAKE_PREFIX_PATH", 0, None), + (r"By not providing \"([a-zA-Z0-9]+).cmake\" in CMAKE_MODULE_PATH this project", 0, None), + (r"C library '(.*)' not found", 0, None), + (r"CMake Error at cmake\/modules\/([a-zA-Z0-9]+).cmake", 0, None), + (r"Can't locate [a-zA-Z0-9_\-\/\.]+ in @INC " r"\(you may need to install the ([a-zA-Z0-9_\-:]+) module\)", 0, 'perl'), + (r"Cannot find ([a-zA-Z0-9\-_\.]*)", 1, None), + (r"Checking for (.*?)\.\.\.no", 0, None), + (r"Checking for (.*?)\s*: not found", 0, None), + (r"Checking for (.*?)\s>=.*\s*: not found", 0, None), + (r"Could not find '([a-zA-Z0-9\-\_]*)' \([~<>=]+ ([0-9.]+).*\) among [0-9]+ total gem", 0, 'ruby'), + (r"Could not find gem '([a-zA-Z0-9\-\_]+) \([~<>=0-9\.\, ]+\) ruby'", 0, 'ruby'), + (r"Could not find suitable distribution for Requirement.parse\('([a-zA-Z\-\.]*)", 0, None), + (r"Download error on https://pypi.python.org/simple/([a-zA-Z0-9\-\._:]+)/", 0, 'pypi'), + (r"Downloading https?://.*\.python\.org/packages/.*/.?/([A-Za-z]*)/.*", 0, None), + (r"ERROR: Could not find a valid gem '([a-zA-Z0-9\-\_\:]*)' \([>=]+ ([0-9.]+).*\)", 0, 'ruby'), + (r"ERROR: dependencies ['‘]([a-zA-Z0-9\-\.]*)['’].* are not available for package ['‘].*['’]", 0, 'R'), + (r"ERROR: dependencies ['‘].*['’], ['‘]([a-zA-Z0-9\-\.]*)['’],.* are not available for package ['‘].*['’]", 0, 'R'), + (r"ERROR: dependencies.*['‘]([a-zA-Z0-9\-\.]*)['’] are not available for package ['‘].*['’]", 0, 'R'), + (r"ERROR: dependency ['‘]([a-zA-Z0-9\-\.]*)['’] is not available for package ['‘].*['’]", 0, 'R'), + (r"Error: Unable to find (.*)", 0, None), + (r"Error: package ['‘]([a-zA-Z0-9\-\.]*)['’] required by", 0, 'R'), + (r"Gem::LoadError: Could not find '([a-zA-Z0-9\-\_]*)'", 0, 'ruby'), + (r"ImportError:.* No module named '?([a-zA-Z0-9\-\._]+)'?", 0, 'pypi'), + (r"ImportError\: ([a-zA-Z]+) module missing", 0, None), + (r"ImportError\: (?:No module|cannot import) named? (.*)", 0, None), + (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:\/\_]+)", 0, 'ruby table'), + (r"LoadError: cannot load such file -- ([a-zA-Z0-9\-:]+)/.*", 0, 'ruby'), + (r"ModuleNotFoundError.*No module named (.*)", 0, None), + (r"Native dependency '(.*)' not found", 0, "pkgconfig"), + (r"No library found for -l([a-zA-Z\-])", 0, None), + (r"No (?:matching distribution|local packages or working download links) found for ([a-zA-Z0-9\-\.\_]+)", 0, 'pypi'), + (r"No package '([a-zA-Z0-9\-:]*)' found", 0, 'pkgconfig'), + (r"No rule to make target `(.*)',", 0, None), + (r"Package '([a-zA-Z0-9\-:]*)', required by '.*', not found", 0, 'pkgconfig'), + (r"Package which this enhances but not available for checking: ['‘]([a-zA-Z0-9\-]*)['’]", 0, 'R'), + (r"Perhaps you should add the directory containing `([a-zA-Z0-9\-:]*)\.pc'", 0, 'pkgconfig'), + (r"Program (.*) found: NO", 0, None), + (r"Target '[a-zA-Z0-9\-]' can't be generated as '(.*)' could not be found", 0, None), + (r"Unable to `import (.*)`", 0, None), + (r"Unable to find '(.*)'", 0, None), + (r"Unknown packages? ['‘]([a-zA-Z0-9\-]*)['’].* in Rd xrefs", 0, 'R'), + (r"WARNING: [a-zA-Z\-\_]+ dependency on ([a-zA-Z0-9\-\_:]*) \([<>=~]+ ([0-9.]+).*\) .*", 0, 'ruby'), + (r"Warning: prerequisite ([a-zA-Z:]+) [0-9\.]+ not found.", 0, 'perl'), + (r"Warning\: no usable ([a-zA-Z0-9]+) found", 0, None), + (r"You need ([a-zA-Z0-9\-\_]*) to build this program.", 1, None), + (r"[Dd]ependency (.*) found: NO \(tried pkgconfig(?: and cmake)?\)", 0, 'pkgconfig'), + (r"[Dd]ependency (.*) found: NO", 0, None), + (r"[a-zA-Z0-9\-:]* is not installed: cannot load such file -- rdoc/([a-zA-Z0-9\-:]*)", 0, 'ruby'), + (r"\-\- Could NOT find ([a-zA-Z0-9]+)", 0, None), + (r"\/bin\/ld: cannot find (-l[a-zA-Z0-9\_]+)", 0, None), + (r"^.*By not providing \"Find(.*).cmake\" in CMAKE_MODULE_PATH this.*$", 0, None), + (r"^.*Could not find a package configuration file provided by \"(.*)\".*$", 0, None), + (r"^.*\"(.*)\" with any of the following names.*$", 0, None), + (r"[Cc]hecking for (.*) (?:support|development files|with pkg-config)?\.\.\. [Nn]o", 0, None), + (r"checking (.*?)\.\.\. no", 0, None), + (r"checking for (.*) in default path\.\.\. not found", 0, None), + (r"checking for (.*)... configure: error", 0, None), + (r"checking for (.*?)\.\.\. no", 0, None), + (r"checking for [a-zA-Z0-9\_\-]+ in (.*?)\.\.\. no", 0, None), + (r"checking for library containing (.*)... no", 0, None), + (r"checking for perl module ([a-zA-Z:]+) [0-9\.]+... no", 0, 'perl'), + (r"configure: error: (?:pkg-config missing|Unable to locate) (.*)", 0, None), + (r"configure: error: ([a-zA-Z0-9]+) (?:is required to build|not found)", 0, None), + (r"configure: error: Cannot find (.*)\. Make sure", 0, None), + (r"fatal error\: (.*)\: No such file or directory", 0, None), + (r"make: ([a-zA-Z0-9].+): Command not found", 0, None), + (r"meson\.build\:[\d]+\:[\d]+\: ERROR: C library \'(.*)\' not found", 0, None), + (r"there is no package called ['‘]([a-zA-Z0-9\-\.]*)['’]", 0, 'R'), + (r"unable to execute '([a-zA-Z\-]*)': No such file or directory", 0, None), + (r"warning: failed to load external entity " r"\"(/usr/share/sgml/docbook/xsl-stylesheets)/.*\"", 0, None), + (r"which\: no ([a-zA-Z\-]*) in \(", 0, None), + (r"you may need to install the ([a-zA-Z0-9_\-:\.]*) module", 0, 'perl'), + ] + def get_metadata_conf(self): + """Gather package metadata from the tarball module.""" + metadata = {} + metadata['name'] = tarball.name + if self.urlban: + metadata['url'] = re.sub(self.urlban, "localhost", tarball.url) + metadata['archives'] = re.sub(self.urlban, "localhost", " ".join(tarball.archives)) + else: + metadata['url'] = tarball.url + metadata['archives'] = " ".join(tarball.archives) -def parse_existing_spec(path, name): - """Determine the old version, old patch list, old keyid, and cves from old spec file.""" - global old_version - global old_patches - global old_keyid - global cves + metadata['giturl'] = tarball.giturl + metadata['domain'] = tarball.domain - spec = os.path.join(path, "{}.spec".format(name)) - if not os.path.exists(spec): - return + if self.alias: + metadata['alias'] = self.alias + else: + metadata['alias'] = "" + return metadata - found_old_version = False - found_old_patches = False - ver_regex = r"^Version *: *(.*) *$" - patch_regex = r"^Patch[0-9]* *: *(.*) *$" + def rewrite_config_opts(self, path): + """Rewrite options.conf file when an option has changed (verify_required for example).""" + config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) + config_f['package'] = self.get_metadata_conf() + config_f['autospec'] = {} - # If git history exists, read the Version and Patch* spec header fields - # from the latest commit to take priority over the working copy. - cmd = ["git", "-C", path, "grep", "-E", "-h", ver_regex, "HEAD", spec] - result = subprocess.run(cmd, capture_output=True) - if result.returncode == 0: - # The first matching line is from the spec header (hopefully) - line = result.stdout.decode().split("\n")[0] - m = re.search(ver_regex, line) - if m: - old_version = m.group(1) - found_old_version = True + # Populate missing configuration options + # (in case of a user-created options.conf) + missing = set(self.config_options.keys()).difference(set(self.config_opts.keys())) + for option in missing: + self.config_opts[option] = False - cmd = ["git", "-C", path, "grep", "-E", "-h", patch_regex, "HEAD", spec] - result = subprocess.run(cmd, capture_output=True) - if result.returncode == 0: - lines = result.stdout.decode().split("\n") - for line in lines: - m = re.search(patch_regex, line) - if m: - old_patches.append(m.group(1).lower()) - found_old_patches = True + for fname, comment in sorted(self.config_options.items()): + config_f.set('autospec', '# {}'.format(comment)) + config_f['autospec'][fname] = 'true' if self.config_opts[fname] else 'false' + write_config(config_f, path) - with open_auto(spec, "r") as inp: - for line in inp.readlines(): - line = line.strip().replace("\r", "").replace("\n", "") - if "Source0 file verified with key" in line: - keyidx = line.find('0x') + 2 - old_keyid = line[keyidx:].split()[0] if keyidx > 2 else old_keyid - # As a fallback, read the Version and Patch* header fields from the - # working copy of the spec, in case a git repo does not exist. - m = re.search(ver_regex, line) - if m and not found_old_version: - old_version = m.group(1) - found_old_version = True - m = re.search(patch_regex, line) - if m and not found_old_patches: - old_patches.append(m.group(1).lower()) + def create_conf(self, path): + """Create options.conf file and use deprecated configuration files or defaults to populate.""" + config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) - # Ignore nopatch - for patch in patches: - patch = patch.lower() - if patch not in old_patches and patch.endswith(".patch") and patch.startswith("cve-"): - cves.append(patch.upper().split(".PATCH")[0]) + # first the metadata + config_f['package'] = self.get_metadata_conf() - -def parse_config_versions(path): - """Parse the versions configuration file.""" - global versions - global parsed_versions - - # Only actually parse it the first time around - if not parsed_versions: - for line in read_conf_file(os.path.join(path, "versions")): - # Simply whitespace-separated fields - fields = line.split() - version = fields.pop(0) - if len(fields): - url = fields.pop(0) + # next the options + config_f['autospec'] = {} + for fname, comment in sorted(self.config_options.items()): + config_f.set('autospec', '# {}'.format(comment)) + if os.path.exists(fname): + config_f['autospec'][fname] = 'true' + os.remove(fname) else: - url = "" - # Catch and report duplicate URLs in the versions file. Don't stop, - # but assume only the first one is valid and drop the rest. - if version in parsed_versions and url != parsed_versions[version]: - print_warning("Already have a URL defined for {}: {}" - .format(version, parsed_versions[version])) - print_warning("Dropping {}, but you should check my work" - .format(url)) - else: - parsed_versions[version] = url - if len(fields): - print_warning("Extra fields detected in `versions` file entry:\n{}" - .format(line)) - print_warning("I'll delete them, but you should check my work") + config_f['autospec'][fname] = 'false' - # We'll combine what we just parsed from the versions file with any other - # versions that have already been defined, most likely the version actually - # provided in the Makefile's URL variable, so we don't drop any. - for version in parsed_versions: - versions[version] = parsed_versions[version] + # default lto to true for new things + config_f['autospec']['use_lto'] = 'true' - return versions + # renamed options need special care + if os.path.exists("skip_test_suite"): + config_f['autospec']['skip_tests'] = 'true' + os.remove("skip_test_suite") + write_config(config_f, path) + def create_buildreq_cache(self, path, version): + """Make the buildreq_cache file.""" + content = self.read_conf_file(os.path.join(path, "buildreq_cache")) + # don't create an empty cache file + if len(buildreq.buildreqs_cache) < 1: + try: + # file was possibly added to git so we should clean it up + os.unlink(content) + except Exception: + pass + return + if not content: + pkgs = sorted(buildreq.buildreqs_cache) + else: + pkgs = sorted(set(content[1:]).union(buildreq.buildreqs_cache)) + with open(os.path.join(path, 'buildreq_cache'), "w") as cachefile: + cachefile.write("\n".join([version] + pkgs)) + self.config_files.add('buildreq_cache') -def parse_config_files(path, bump, filemanager, version): - """Parse the various configuration files that may exist in the package directory.""" - global extra_configure - global extra_configure32 - global extra_configure64 - global extra_configure_avx2 - global extra_configure_avx512 - global extra_configure_openmpi - global config_files - global extra_sources - global parallel_build - global license_fetch - global license_show - global git_uri - global os_packages - global urlban - global config_file - global profile_payload - global config_opts - global extra_make - global extra32_make - global extra_make_install - global extra_make32_install - global extra_cmake - global extra_cmake_openmpi - global cmake_srcdir - global subdir - global install_macro - global disable_static - global prep_prepend - global build_prepend - global build_append - global make_prepend - global install_prepend - global install_append - global service_restart - global patches - global autoreconf - global set_gopath - global yum_conf - global custom_desc - global custom_summ - global failed_pattern_dir - global versions + def create_versions(self, path, versions): + """Make versions file.""" + with open(os.path.join(path, "versions"), 'w') as vfile: + for version in versions: + vfile.write(version) + if versions[version]: + vfile.write('\t' + versions[version]) + vfile.write('\n') + self.config_files.add("versions") - packages_file = None + def read_config_opts(self, path): + """Read config options from path/options.conf.""" + opts_path = os.path.join(path, 'options.conf') + if not os.path.exists(opts_path): + self.create_conf(path) - # Require autospec.conf for additional features - if os.path.exists(config_file): - config = configparser.ConfigParser(interpolation=None) - config.read(config_file) - - if "autospec" not in config.sections(): - print("Missing autospec section..") + config_f = configparser.ConfigParser(interpolation=None) + config_f.read(opts_path) + if "autospec" not in config_f.sections(): + print("Missing autospec section in options.conf") sys.exit(1) - git_uri = config['autospec'].get('git', None) - license_fetch = config['autospec'].get('license_fetch', None) - license_show = config['autospec'].get('license_show', None) - packages_file = config['autospec'].get('packages_file', None) - yum_conf = config['autospec'].get('yum_conf', None) - failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None) + if 'package' in config_f.sections() and config_f['package'].get('alias'): + self.alias = config_f['package']['alias'] - # support reading the local files relative to config_file - if packages_file and not os.path.isabs(packages_file): - packages_file = os.path.join(os.path.dirname(config_file), packages_file) - if yum_conf and not os.path.isabs(yum_conf): - yum_conf = os.path.join(os.path.dirname(config_file), yum_conf) - if failed_pattern_dir and not os.path.isabs(failed_pattern_dir): - failed_pattern_dir = os.path.join(os.path.dirname(config_file), failed_pattern_dir) + for key in config_f['autospec']: + self.config_opts[key] = config_f['autospec'].getboolean(key) - if not packages_file: - print("Warning: Set [autospec][packages_file] path to package list file for " - "requires validation") - packages_file = os.path.join(os.path.dirname(config_file), "packages") + # Rewrite the configuration file in case of formatting changes since a + # configuration file may exist without any comments (either due to an older + # version of autospec or if it was user-created) + self.rewrite_config_opts(path) - urlban = config['autospec'].get('urlban', None) + # Don't use the ChangeLog files if the giturl is set + # ChangeLog is just extra noise when we can already see the gitlog + if "package" in config_f.sections() and config_f['package'].get('giturl'): + keys = [] + for k, v in self.transforms.items(): + if v == "ChangeLog": + keys.append(k) + for k in keys: + self.transforms.pop(k) - # Read values from options.conf (and deprecated files) and rewrite as necessary - read_config_opts(path) + def read_file(self, path, track=True): + """Read full file at path. - if not git_uri: - print("Warning: Set [autospec][git] upstream template for remote git URI configuration") - if not license_fetch: - print("Warning: Set [autospec][license_fetch] uri for license fetch support") - if not license_show: - print("Warning: Set [autospec][license_show] uri for license link check support") - if not yum_conf: - print("Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation") - yum_conf = os.path.join(os.path.dirname(config_file), "image-creator/yum.conf") + If the file does not exist (or is not expected to exist) + in the package git repo, specify 'track=False'. + """ + try: + with open(path, "r") as f: + if track: + self.config_files.add(os.path.basename(path)) + return f.readlines() + except EnvironmentError: + return [] - if packages_file: - os_packages = set(read_conf_file(packages_file, track=False)) - else: - os_packages = set(read_conf_file("~/packages", track=False)) + def read_conf_file(self, path, track=True): + """Read configuration file at path. - wrapper = textwrap.TextWrapper() - wrapper.initial_indent = "# " - wrapper.subsequent_indent = "# " + If the config file does not exist (or is not expected to exist) + in the package git repo, specify 'track=False'. + """ + lines = self.read_file(path, track=track) + return [l.strip() for l in lines if not l.strip().startswith("#") and l.split()] - def write_default_conf_file(name, description): + def read_script_file(self, path, track=True): + """Read RPM script snippet file at path. + + Returns verbatim, except for possibly the first line. + + If the config file does not exist (or is not expected to exist) + in the package git repo, specify 'track=False'. + """ + lines = self.read_file(path, track=track) + if len(lines) > 0 and (lines[0].startswith('#!') or lines[0].startswith('# -*- ')): + lines = lines[1:] + # Remove any trailing whitespace and newlines. The newlines are later + # restored by writer functions. + return [line.rstrip() for line in lines] + + def setup_patterns(self, path=None): + """Read each pattern configuration file and assign to the appropriate variable.""" + read_pattern_conf("ignored_commands", self.ignored_commands, list_format=True, path=path) + read_pattern_conf("failed_commands", self.failed_commands, path=path) + read_pattern_conf("maven_jars", self.maven_jars, path=path) + read_pattern_conf("gems", self.gems, path=path) + read_pattern_conf("license_hashes", self.license_hashes, path=path) + read_pattern_conf("license_translations", self.license_translations, path=path) + read_pattern_conf("license_blacklist", self.license_blacklist, list_format=True, path=path) + read_pattern_conf("qt_modules", self.qt_modules, path=path) + read_pattern_conf("cmake_modules", self.cmake_modules, path=path) + + def parse_existing_spec(self, path, name): + """Determine the old version, old patch list, old keyid, and cves from old spec file.""" + spec = os.path.join(path, "{}.spec".format(name)) + if not os.path.exists(spec): + return + + found_old_version = False + found_old_patches = False + ver_regex = r"^Version *: *(.*) *$" + patch_regex = r"^Patch[0-9]* *: *(.*) *$" + + # If git history exists, read the Version and Patch* spec header fields + # from the latest commit to take priority over the working copy. + cmd = ["git", "-C", path, "grep", "-E", "-h", ver_regex, "HEAD", spec] + result = subprocess.run(cmd, capture_output=True) + if result.returncode == 0: + # The first matching line is from the spec header (hopefully) + line = result.stdout.decode().split("\n")[0] + m = re.search(ver_regex, line) + if m: + self.old_version = m.group(1) + found_old_version = True + + cmd = ["git", "-C", path, "grep", "-E", "-h", patch_regex, "HEAD", spec] + result = subprocess.run(cmd, capture_output=True) + if result.returncode == 0: + lines = result.stdout.decode().split("\n") + for line in lines: + m = re.search(patch_regex, line) + if m: + self.old_patches.append(m.group(1).lower()) + found_old_patches = True + + with open_auto(spec, "r") as inp: + for line in inp.readlines(): + line = line.strip().replace("\r", "").replace("\n", "") + if "Source0 file verified with key" in line: + keyidx = line.find('0x') + 2 + self.old_keyid = line[keyidx:].split()[0] if keyidx > 2 else self.old_keyid + # As a fallback, read the Version and Patch* header fields from the + # working copy of the spec, in case a git repo does not exist. + m = re.search(ver_regex, line) + if m and not found_old_version: + self.old_version = m.group(1) + found_old_version = True + m = re.search(patch_regex, line) + if m and not found_old_patches: + self.old_patches.append(m.group(1).lower()) + + # Ignore nopatch + for patch in self.patches: + patch = patch.lower() + if patch not in self.old_patches and patch.endswith(".patch") and patch.startswith("cve-"): + self.cves.append(patch.upper().split(".PATCH")[0]) + + def parse_config_versions(self, path): + """Parse the versions configuration file.""" + # Only actually parse it the first time around + if not self.parsed_versions: + for line in self.read_conf_file(os.path.join(path, "versions")): + # Simply whitespace-separated fields + fields = line.split() + version = fields.pop(0) + if len(fields): + url = fields.pop(0) + else: + url = "" + # Catch and report duplicate URLs in the versions file. Don't stop, + # but assume only the first one is valid and drop the rest. + if version in self.parsed_versions and url != self.parsed_versions[version]: + print_warning("Already have a URL defined for {}: {}" + .format(version, self.parsed_versions[version])) + print_warning("Dropping {}, but you should check my work" + .format(url)) + else: + self.parsed_versions[version] = url + if len(fields): + print_warning("Extra fields detected in `versions` file entry:\n{}" + .format(line)) + print_warning("I'll delete them, but you should check my work") + + # We'll combine what we just parsed from the versions file with any other + # versions that have already been defined, most likely the version actually + # provided in the Makefile's URL variable, so we don't drop any. + for version in self.parsed_versions: + self.versions[version] = self.parsed_versions[version] + + return self.versions + + def write_default_conf_file(self, path, name, wrapper, description): """Write default configuration file with description to file name.""" - config_files.add(name) + self.config_files.add(name) filename = os.path.join(path, name) if os.path.isfile(filename): return write_out(filename, wrapper.fill(description) + "\n") - write_default_conf_file("buildreq_ban", - "This file contains build requirements that get picked up but are " - "undesirable. One entry per line, no whitespace.") - write_default_conf_file("pkgconfig_ban", - "This file contains pkgconfig build requirements that get picked up but" - " are undesirable. One entry per line, no whitespace.") - write_default_conf_file("requires_ban", - "This file contains runtime requirements that get picked up but are " - "undesirable. One entry per line, no whitespace.") - write_default_conf_file("buildreq_add", - "This file contains additional build requirements that did not get " - "picked up automatically. One name per line, no whitespace.") - write_default_conf_file("pkgconfig_add", - "This file contains additional pkgconfig build requirements that did " - "not get picked up automatically. One name per line, no whitespace.") - write_default_conf_file("requires_add", - "This file contains additional runtime requirements that did not get " - "picked up automatically. One name per line, no whitespace.") - write_default_conf_file("excludes", - "This file contains the output files that need %exclude. Full path " - "names, one per line.") + def parse_config_files(self, path, bump, filemanager, version): + """Parse the various configuration files that may exist in the package directory.""" + packages_file = None - content = read_conf_file(os.path.join(path, "release")) - if content and content[0]: - r = int(content[0]) - if bump: - r += 1 - tarball.release = str(r) - print("Release :", tarball.release) + # Require autospec.conf for additional features + if os.path.exists(self.config_file): + config = configparser.ConfigParser(interpolation=None) + config.read(self.config_file) - content = read_conf_file(os.path.join(path, "extra_sources")) - for source in content: - fields = source.split(maxsplit=1) - print("Adding additional source file: %s" % fields[0]) - config_files.add(os.path.basename(fields[0])) - extra_sources.append(fields) + if "autospec" not in config.sections(): + print("Missing autospec section..") + sys.exit(1) - content = read_conf_file(os.path.join(path, "buildreq_ban")) - for banned in content: - print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + self.git_uri = config['autospec'].get('git', None) + self.license_fetch = config['autospec'].get('license_fetch', None) + self.license_show = config['autospec'].get('license_show', None) + packages_file = config['autospec'].get('packages_file', None) + self.yum_conf = config['autospec'].get('yum_conf', None) + self.failed_pattern_dir = config['autospec'].get('failed_pattern_dir', None) - content = read_conf_file(os.path.join(path, "pkgconfig_ban")) - for banned in content: - banned = "pkgconfig(%s)" % banned - print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + # support reading the local files relative to config_file + if packages_file and not os.path.isabs(packages_file): + packages_file = os.path.join(os.path.dirname(self.config_file), packages_file) + if self.yum_conf and not os.path.isabs(self.yum_conf): + self.yum_conf = os.path.join(os.path.dirname(self.config_file), self.yum_conf) + if self.failed_pattern_dir and not os.path.isabs(self.failed_pattern_dir): + self.failed_pattern_dir = os.path.join(os.path.dirname(self.config_file), self.failed_pattern_dir) - content = read_conf_file(os.path.join(path, "requires_ban")) - for banned in content: - print("Banning runtime requirement: %s." % banned) - buildreq.banned_requires.add(banned) - buildreq.requires.discard(banned) + if not packages_file: + print("Warning: Set [autospec][packages_file] path to package list file for " + "requires validation") + packages_file = os.path.join(os.path.dirname(self.config_file), "packages") - content = read_conf_file(os.path.join(path, "buildreq_add")) - for extra in content: - print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + self.urlban = config['autospec'].get('urlban', None) - cache_file = os.path.join(path, "buildreq_cache") - content = read_conf_file(cache_file) - if content and content[0] == version: - for extra in content[1:]: - print("Adding additional build (cache) requirement: %s." % extra) + # Read values from options.conf (and deprecated files) and rewrite as necessary + self.read_config_opts(path) + + if not self.git_uri: + print("Warning: Set [autospec][git] upstream template for remote git URI configuration") + if not self.license_fetch: + print("Warning: Set [autospec][license_fetch] uri for license fetch support") + if not self.license_show: + print("Warning: Set [autospec][license_show] uri for license link check support") + if not self.yum_conf: + print("Warning: Set [autospec][yum_conf] path to yum.conf file for whatrequires validation") + self.yum_conf = os.path.join(os.path.dirname(self.config_file), "image-creator/yum.conf") + + if packages_file: + self.os_packages = set(self.read_conf_file(packages_file, track=False)) + else: + self.os_packages = set(self.read_conf_file("~/packages", track=False)) + + wrapper = textwrap.TextWrapper() + wrapper.initial_indent = "# " + wrapper.subsequent_indent = "# " + + self.write_default_conf_file(path, "buildreq_ban", wrapper, + "This file contains build requirements that get picked up but are " + "undesirable. One entry per line, no whitespace.") + self.write_default_conf_file(path, "pkgconfig_ban", wrapper, + "This file contains pkgconfig build requirements that get picked up but" + " are undesirable. One entry per line, no whitespace.") + self.write_default_conf_file(path, "requires_ban", wrapper, + "This file contains runtime requirements that get picked up but are " + "undesirable. One entry per line, no whitespace.") + self.write_default_conf_file(path, "buildreq_add", wrapper, + "This file contains additional build requirements that did not get " + "picked up automatically. One name per line, no whitespace.") + self.write_default_conf_file(path, "pkgconfig_add", wrapper, + "This file contains additional pkgconfig build requirements that did " + "not get picked up automatically. One name per line, no whitespace.") + self.write_default_conf_file(path, "requires_add", wrapper, + "This file contains additional runtime requirements that did not get " + "picked up automatically. One name per line, no whitespace.") + self.write_default_conf_file(path, "excludes", wrapper, + "This file contains the output files that need %exclude. Full path " + "names, one per line.") + + content = self.read_conf_file(os.path.join(path, "release")) + if content and content[0]: + r = int(content[0]) + if bump: + r += 1 + tarball.release = str(r) + print("Release :", tarball.release) + + content = self.read_conf_file(os.path.join(path, "extra_sources")) + for source in content: + fields = source.split(maxsplit=1) + print("Adding additional source file: %s" % fields[0]) + self.config_files.add(os.path.basename(fields[0])) + self.extra_sources.append(fields) + + content = self.read_conf_file(os.path.join(path, "buildreq_ban")) + for banned in content: + print("Banning build requirement: %s." % banned) + buildreq.banned_buildreqs.add(banned) + buildreq.buildreqs.discard(banned) + buildreq.buildreqs_cache.discard(banned) + + content = self.read_conf_file(os.path.join(path, "pkgconfig_ban")) + for banned in content: + banned = "pkgconfig(%s)" % banned + print("Banning build requirement: %s." % banned) + buildreq.banned_buildreqs.add(banned) + buildreq.buildreqs.discard(banned) + buildreq.buildreqs_cache.discard(banned) + + content = self.read_conf_file(os.path.join(path, "requires_ban")) + for banned in content: + print("Banning runtime requirement: %s." % banned) + buildreq.banned_requires.add(banned) + buildreq.requires.discard(banned) + + content = self.read_conf_file(os.path.join(path, "buildreq_add")) + for extra in content: + print("Adding additional build requirement: %s." % extra) buildreq.add_buildreq(extra) - else: - try: - os.unlink(cache_file) - except FileNotFoundError: - pass - except Exception as e: - print_warning(f"Unable to remove buildreq_cache file: {e}") - content = read_conf_file(os.path.join(path, "pkgconfig_add")) - for extra in content: - extra = "pkgconfig(%s)" % extra - print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + cache_file = os.path.join(path, "buildreq_cache") + content = self.read_conf_file(cache_file) + if content and content[0] == version: + for extra in content[1:]: + print("Adding additional build (cache) requirement: %s." % extra) + buildreq.add_buildreq(extra) + else: + try: + os.unlink(cache_file) + except FileNotFoundError: + pass + except Exception as e: + print_warning(f"Unable to remove buildreq_cache file: {e}") - content = read_conf_file(os.path.join(path, "requires_add")) - for extra in content: - print("Adding additional runtime requirement: %s." % extra) - buildreq.add_requires(extra, override=True) + content = self.read_conf_file(os.path.join(path, "pkgconfig_add")) + for extra in content: + extra = "pkgconfig(%s)" % extra + print("Adding additional build requirement: %s." % extra) + buildreq.add_buildreq(extra) - content = read_conf_file(os.path.join(path, "excludes")) - for exclude in content: - print("%%exclude for: %s." % exclude) - filemanager.excludes += content + content = self.read_conf_file(os.path.join(path, "requires_add")) + for extra in content: + print("Adding additional runtime requirement: %s." % extra) + buildreq.add_requires(extra, self.os_packages, override=True) - content = read_conf_file(os.path.join(path, "extras")) - for extra in content: - print("extras for : %s." % extra) - filemanager.extras += content + content = self.read_conf_file(os.path.join(path, "excludes")) + for exclude in content: + print("%%exclude for: %s." % exclude) + filemanager.excludes += content - for fname in os.listdir(path): - if not re.search('.+_extras$', fname) or fname == "dev_extras": - continue - content = {} - content['files'] = read_conf_file(os.path.join(path, fname)) - if not content: - print_warning(f"Error reading custom extras file: {fname}") - continue - req_file = os.path.join(path, f'{fname}_requires') - if os.path.isfile(req_file): - content['requires'] = read_conf_file(req_file) - name = fname[:-len("_extras")] - print(f"extras-{name} for {content['files']}") - filemanager.custom_extras["extras-" + f"{name}"] = content + content = self.read_conf_file(os.path.join(path, "extras")) + for extra in content: + print("extras for : %s." % extra) + filemanager.extras += content - content = read_conf_file(os.path.join(path, "dev_extras")) - for extra in content: - print("dev for : %s." % extra) - filemanager.dev_extras += content + for fname in os.listdir(path): + if not re.search('.+_extras$', fname) or fname == "dev_extras": + continue + content = {} + content['files'] = self.read_conf_file(os.path.join(path, fname)) + if not content: + print_warning(f"Error reading custom extras file: {fname}") + continue + req_file = os.path.join(path, f'{fname}_requires') + if os.path.isfile(req_file): + content['requires'] = self.read_conf_file(req_file) + name = fname[:-len("_extras")] + print(f"extras-{name} for {content['files']}") + filemanager.custom_extras["extras-" + f"{name}"] = content - content = read_conf_file(os.path.join(path, "setuid")) - for suid in content: - print("setuid for : %s." % suid) - filemanager.setuid += content + content = self.read_conf_file(os.path.join(path, "dev_extras")) + for extra in content: + print("dev for : %s." % extra) + filemanager.dev_extras += content - content = read_conf_file(os.path.join(path, "attrs")) - for line in content: - attr = line.split() - filename = attr.pop() - print("%attr({0},{1},{2}) for: {3}".format( - attr[0], attr[1], attr[2], filename)) - filemanager.attrs[filename] = attr + content = self.read_conf_file(os.path.join(path, "setuid")) + for suid in content: + print("setuid for : %s." % suid) + filemanager.setuid += content - patches += read_conf_file(os.path.join(path, "series")) - pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches] - cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(aclocal.m4)|(configure.ac|configure.in))\" %s" % " ".join(pfiles) # noqa: W605 - if patches and call(cmd, - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0: - autoreconf = True + content = self.read_conf_file(os.path.join(path, "attrs")) + for line in content: + attr = line.split() + filename = attr.pop() + print("%attr({0},{1},{2}) for: {3}".format( + attr[0], attr[1], attr[2], filename)) + filemanager.attrs[filename] = attr - # Parse the version-specific patch lists - update_security_sensitive = False - for version in versions: - verpatches[version] = read_conf_file(os.path.join(path, '.'.join(['series', version]))) - if any(p.lower().startswith('cve-') for p in verpatches[version]): + self.patches += self.read_conf_file(os.path.join(path, "series")) + pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in self.patches] + cmd = "egrep \"(\+\+\+|\-\-\-).*((Makefile.am)|(aclocal.m4)|(configure.ac|configure.in))\" %s" % " ".join(pfiles) # noqa: W605 + if self.patches and call(cmd, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0: + self.autoreconf = True + + # Parse the version-specific patch lists + update_security_sensitive = False + for version in self.versions: + self.verpatches[version] = self.read_conf_file(os.path.join(path, '.'.join(['series', version]))) + if any(p.lower().startswith('cve-') for p in self.verpatches[version]): + update_security_sensitive = True + + if any(p.lower().startswith('cve-') for p in self.patches): update_security_sensitive = True - if any(p.lower().startswith('cve-') for p in patches): - update_security_sensitive = True + if update_security_sensitive: + self.config_opts['security_sensitive'] = True + self.rewrite_config_opts(path) - if update_security_sensitive: - config_opts['security_sensitive'] = True - rewrite_config_opts(path) + content = self.read_conf_file(os.path.join(path, "configure")) + self.extra_configure = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure")) - extra_configure = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure32")) + self.extra_configure32 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure32")) - extra_configure32 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure64")) + self.extra_configure64 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure64")) - extra_configure64 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure_avx2")) + self.extra_configure_avx2 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure_avx2")) - extra_configure_avx2 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure_avx512")) + self.extra_configure_avx512 = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure_avx512")) - extra_configure_avx512 = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "configure_openmpi")) + self.extra_configure_openmpi = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "configure_openmpi")) - extra_configure_openmpi = " \\\n".join(content) + if self.config_opts["keepstatic"]: + self.disable_static = "" + if self.config_opts['broken_parallel_build']: + self.parallel_build = "" - if config_opts["keepstatic"]: - disable_static = "" - if config_opts['broken_parallel_build']: - parallel_build = "" + content = self.read_conf_file(os.path.join(path, "make_args")) + if content: + self.extra_make = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make_args")) - if content: - extra_make = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "make32_args")) + if content: + self.extra32_make = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make32_args")) - if content: - extra32_make = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "make_install_args")) + if content: + self.extra_make_install = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make_install_args")) - if content: - extra_make_install = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "make32_install_args")) + if content: + self.extra_make32_install = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "make32_install_args")) - if content: - extra_make32_install = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "install_macro")) + if content and content[0]: + self.install_macro = content[0] - content = read_conf_file(os.path.join(path, "install_macro")) - if content and content[0]: - install_macro = content[0] + content = self.read_conf_file(os.path.join(path, "cmake_args")) + if content: + self.extra_cmake = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "cmake_args")) - if content: - extra_cmake = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "cmake_args_openmpi")) + if content: + self.extra_cmake_openmpi = " \\\n".join(content) - content = read_conf_file(os.path.join(path, "cmake_args_openmpi")) - if content: - extra_cmake_openmpi = " \\\n".join(content) + content = self.read_conf_file(os.path.join(path, "cmake_srcdir")) + if content and content[0]: + self.cmake_srcdir = content[0] - content = read_conf_file(os.path.join(path, "cmake_srcdir")) - if content and content[0]: - cmake_srcdir = content[0] + content = self.read_conf_file(os.path.join(path, "subdir")) + if content and content[0]: + self.subdir = content[0] - content = read_conf_file(os.path.join(path, "subdir")) - if content and content[0]: - subdir = content[0] + content = self.read_conf_file(os.path.join(path, "build_pattern")) + if content and content[0]: + buildpattern.set_build_pattern(content[0], 20) + self.autoreconf = False - content = read_conf_file(os.path.join(path, "build_pattern")) - if content and content[0]: - buildpattern.set_build_pattern(content[0], 20) - autoreconf = False + content = self.read_script_file(os.path.join(path, "make_check_command")) + if content: + check.tests_config = '\n'.join(content) - content = read_script_file(os.path.join(path, "make_check_command")) - if content: - check.tests_config = '\n'.join(content) + content = self.read_conf_file(os.path.join(path, tarball.name + ".license")) + if content and content[0]: + words = content[0].split() + for word in words: + if word.find(":") < 0: + if not license.add_license(word, self.license_translations, self.license_blacklist): + print_warning("{}: blacklisted license {} ignored.".format(tarball.name + ".license", word)) - content = read_conf_file(os.path.join(path, tarball.name + ".license")) - if content and content[0]: - words = content[0].split() - for word in words: - if word.find(":") < 0: - if not license.add_license(word): - print_warning("{}: blacklisted license {} ignored.".format(tarball.name + ".license", word)) + content = self.read_conf_file(os.path.join(path, "golang_libpath")) + if content and content[0]: + tarball.golibpath = content[0] + print("golibpath : {}".format(tarball.golibpath)) - content = read_conf_file(os.path.join(path, "golang_libpath")) - if content and content[0]: - tarball.golibpath = content[0] - print("golibpath : {}".format(tarball.golibpath)) + if self.config_opts['use_clang']: + self.config_opts['funroll-loops'] = False + buildreq.add_buildreq("llvm") - if config_opts['use_clang']: - config_opts['funroll-loops'] = False - buildreq.add_buildreq("llvm") + if self.config_opts['32bit']: + buildreq.add_buildreq("glibc-libc32") + buildreq.add_buildreq("glibc-dev32") + buildreq.add_buildreq("gcc-dev32") + buildreq.add_buildreq("gcc-libgcc32") + buildreq.add_buildreq("gcc-libstdc++32") - if config_opts['32bit']: - buildreq.add_buildreq("glibc-libc32") - buildreq.add_buildreq("glibc-dev32") - buildreq.add_buildreq("gcc-dev32") - buildreq.add_buildreq("gcc-libgcc32") - buildreq.add_buildreq("gcc-libstdc++32") + if self.config_opts['openmpi']: + buildreq.add_buildreq("openmpi-dev") + buildreq.add_buildreq("modules") + # MPI testsuites generally require "openssh" + buildreq.add_buildreq("openssh") - if config_opts['openmpi']: - buildreq.add_buildreq("openmpi-dev") - buildreq.add_buildreq("modules") - # MPI testsuites generally require "openssh" - buildreq.add_buildreq("openssh") + self.prep_prepend = self.read_script_file(os.path.join(path, "prep_prepend")) + if os.path.isfile(os.path.join(path, "prep_append")): + os.rename(os.path.join(path, "prep_append"), os.path.join(path, "build_prepend")) + self.make_prepend = self.read_script_file(os.path.join(path, "make_prepend")) + self.build_prepend = self.read_script_file(os.path.join(path, "build_prepend")) + self.build_append = self.read_script_file(os.path.join(path, "build_append")) + self.install_prepend = self.read_script_file(os.path.join(path, "install_prepend")) + if os.path.isfile(os.path.join(path, "make_install_append")): + os.rename(os.path.join(path, "make_install_append"), os.path.join(path, "install_append")) + self.install_append = self.read_script_file(os.path.join(path, "install_append")) + self.service_restart = self.read_conf_file(os.path.join(path, "service_restart")) - prep_prepend = read_script_file(os.path.join(path, "prep_prepend")) - if os.path.isfile(os.path.join(path, "prep_append")): - os.rename(os.path.join(path, "prep_append"), os.path.join(path, "build_prepend")) - make_prepend = read_script_file(os.path.join(path, "make_prepend")) - build_prepend = read_script_file(os.path.join(path, "build_prepend")) - build_append = read_script_file(os.path.join(path, "build_append")) - install_prepend = read_script_file(os.path.join(path, "install_prepend")) - if os.path.isfile(os.path.join(path, "make_install_append")): - os.rename(os.path.join(path, "make_install_append"), os.path.join(path, "install_append")) - install_append = read_script_file(os.path.join(path, "install_append")) - service_restart = read_conf_file(os.path.join(path, "service_restart")) + self.profile_payload = self.read_script_file(os.path.join(path, "profile_payload")) - profile_payload = read_script_file(os.path.join(path, "profile_payload")) + self.custom_desc = self.read_conf_file(os.path.join(path, "description")) + self.custom_summ = self.read_conf_file(os.path.join(path, "summary")) - custom_desc = read_conf_file(os.path.join(path, "description")) - custom_summ = read_conf_file(os.path.join(path, "summary")) - - -def load_specfile(specfile): - """Load specfile object with configuration.""" - specfile.urlban = urlban - specfile.keepstatic = config_opts['keepstatic'] - specfile.no_autostart = config_opts['no_autostart'] - specfile.extra_make = extra_make - specfile.extra32_make = extra32_make - specfile.extra_make_install = extra_make_install - specfile.extra_make32_install = extra_make32_install - specfile.extra_cmake = extra_cmake - specfile.extra_cmake_openmpi = extra_cmake_openmpi - specfile.cmake_srcdir = cmake_srcdir or specfile.cmake_srcdir - specfile.subdir = subdir - specfile.install_macro = install_macro - specfile.disable_static = disable_static - specfile.prep_prepend = prep_prepend - specfile.build_prepend = build_prepend - specfile.build_append = build_append - specfile.make_prepend = make_prepend - specfile.install_prepend = install_prepend - specfile.install_append = install_append - specfile.service_restart = service_restart - specfile.extra_sources = extra_sources - specfile.patches = patches - specfile.verpatches = verpatches - specfile.autoreconf = autoreconf - specfile.set_gopath = set_gopath + def load_specfile(self, specfile): + """Load specfile object with configuration.""" + specfile.urlban = self.urlban + specfile.keepstatic = self.config_opts['keepstatic'] + specfile.no_autostart = self.config_opts['no_autostart'] + specfile.extra_make = self.extra_make + specfile.extra32_make = self.extra32_make + specfile.extra_make_install = self.extra_make_install + specfile.extra_make32_install = self.extra_make32_install + specfile.extra_cmake = self.extra_cmake + specfile.extra_cmake_openmpi = self.extra_cmake_openmpi + specfile.cmake_srcdir = self.cmake_srcdir or specfile.cmake_srcdir + specfile.subdir = self.subdir + specfile.install_macro = self.install_macro + specfile.disable_static = self.disable_static + specfile.prep_prepend = self.prep_prepend + specfile.build_prepend = self.build_prepend + specfile.build_append = self.build_append + specfile.make_prepend = self.make_prepend + specfile.install_prepend = self.install_prepend + specfile.install_append = self.install_append + specfile.service_restart = self.service_restart + specfile.extra_sources = self.extra_sources + specfile.patches = self.patches + specfile.verpatches = self.verpatches + specfile.autoreconf = self.autoreconf + specfile.set_gopath = self.set_gopath diff --git a/autospec/files.py b/autospec/files.py index 437ef8f..b6add8f 100644 --- a/autospec/files.py +++ b/autospec/files.py @@ -24,7 +24,6 @@ import re from collections import OrderedDict import build -import config import tarball import util @@ -32,8 +31,9 @@ import util class FileManager(object): """Class to handle spec file %files section management.""" - def __init__(self): + def __init__(self, config): """Set defaults for FileManager.""" + self.config = config self.packages = OrderedDict() # per sub-package file list for spec purposes self.files = set() # global file set to weed out dupes self.files_blacklist = set() @@ -89,7 +89,7 @@ class FileManager(object): def compat_exclude(self, filename): """Exclude non-library files if the package is for compatability.""" - if not config.config_opts.get("compat"): + if not self.config.config_opts.get("compat"): return False patterns = [ @@ -228,8 +228,8 @@ class FileManager(object): # if configured to do so, add .so files to the lib package instead of # the dev package. THis is useful for packages with a plugin # architecture like elfutils and mesa. - so_dest = 'lib' if config.config_opts.get('so_to_lib') else 'dev' - so_dest_ompi = 'openmpi' if config.config_opts.get('so_to_lib') else 'dev' + so_dest = 'lib' if self.config.config_opts.get('so_to_lib') else 'dev' + so_dest_ompi = 'openmpi' if self.config.config_opts.get('so_to_lib') else 'dev' patterns = [ # Patterns for matching files, format is a tuple as follows: diff --git a/autospec/git.py b/autospec/git.py index 6785164..5e08558 100644 --- a/autospec/git.py +++ b/autospec/git.py @@ -25,12 +25,11 @@ import subprocess import build import buildpattern -import config import tarball from util import call, write_out -def commit_to_git(path): +def commit_to_git(path, config): """Update package's git tree for autospec managed changes.""" call("git init", stdout=subprocess.DEVNULL, cwd=path) diff --git a/autospec/license.py b/autospec/license.py index 0e605f0..ec809c0 100644 --- a/autospec/license.py +++ b/autospec/license.py @@ -28,7 +28,6 @@ import sys import urllib.parse import chardet -import config import download import tarball @@ -41,17 +40,17 @@ license_files = [] hashes = dict() -def process_licenses(lics): +def process_licenses(lics, translations, blacklist): """Handle licenses string from the license server. The license server response may contain multiple space-separated licenses. Add each license individually. """ for lic in lics.split(): - add_license(lic) + add_license(lic, translations, blacklist) -def add_license(lic): +def add_license(lic, translations, blacklist): """Add licenses from the server. Add license from license string lic after checking for duplication or @@ -64,10 +63,10 @@ def add_license(lic): result = False # Translate the license if a translation exists - real_lic_str = config.license_translations.get(lic, lic) + real_lic_str = translations.get(lic, lic) real_lics = real_lic_str.split() for real_lic in real_lics: - if real_lic in config.license_blacklist: + if real_lic in blacklist: continue elif real_lic in licenses: result = True @@ -96,7 +95,7 @@ def decode_license(license): return try_with_charset(license, chardet.detect(license)['encoding']) -def license_from_copying_hash(copying, srcdir): +def license_from_copying_hash(copying, srcdir, config): """Add licenses based on the hash of the copying file.""" try: data = get_contents(copying) @@ -124,7 +123,7 @@ def license_from_copying_hash(copying, srcdir): page = response.decode('utf-8').strip() if page: print("License : ", page, " (server) (", hash_sum, ")") - process_licenses(page) + process_licenses(page, config.license_translations, config.license_blacklist) if page != "none": # Strip the build source directory off the front @@ -139,7 +138,9 @@ def license_from_copying_hash(copying, srcdir): return if hash_sum in config.license_hashes: - add_license(config.license_hashes[hash_sum]) + add_license(config.license_hashes[hash_sum], + config.license_translations, + config.license_blacklist) else: if not config.license_show: return @@ -148,7 +149,7 @@ def license_from_copying_hash(copying, srcdir): print_warning("Visit {0} to enter".format(hash_url)) -def scan_for_licenses(srcdir): +def scan_for_licenses(srcdir, config): """Scan the project directory for things we can use to guess a description and summary.""" targets = ["copyright", "copyright.txt", @@ -167,7 +168,8 @@ def scan_for_licenses(srcdir): for dirpath, dirnames, files in os.walk(srcdir): for name in files: if name.lower() in targets or target_pat.search(name.lower()): - license_from_copying_hash(os.path.join(dirpath, name), srcdir) + license_from_copying_hash(os.path.join(dirpath, name), + srcdir, config) if not licenses: print_fatal(" Cannot find any license or a valid {}.license file!\n".format(tarball.name)) diff --git a/autospec/pkg_integrity.py b/autospec/pkg_integrity.py index 2671269..5f28d57 100644 --- a/autospec/pkg_integrity.py +++ b/autospec/pkg_integrity.py @@ -210,6 +210,7 @@ class Verifier(object): """Set default values.""" self.url = kwargs.get('url', None) self.package_sign_path = kwargs.get('package_sign_path', None) + self.config = kwargs.get('config', None) print(SEPT) @staticmethod @@ -569,13 +570,13 @@ class GPGVerifier(Verifier): EMAIL = get_email(pubkey_loc) sign_status = verify_cli(pubkey_loc, self.package_path, self.package_sign_path) if not sign_status: - if config.old_keyid: - compare_keys(KEYID_TRY, config.old_keyid) + if self.config.old_keyid: + compare_keys(KEYID_TRY, self.config.old_keyid) self.print_result(self.package_path) KEYID = KEYID_TRY - config.signature = self.key_url - config.config_opts['verify_required'] = True - config.rewrite_config_opts(os.path.dirname(self.package_path)) + self.config.signature = self.key_url + self.config.config_opts['verify_required'] = True + self.config.rewrite_config_opts(os.path.dirname(self.package_path)) return True else: self.print_result(False, err_msg=sign_status.strerror) @@ -833,14 +834,17 @@ def apply_verification(verifier, **kwargs): return v.verify() -def from_disk(url, package_path, package_check, interactive=True): +def from_disk(url, package_path, package_check, config, interactive=True): """Run verification.""" verifier = get_verifier(package_path) - return apply_verification(verifier, **{ - 'package_path': package_path, - 'package_check': package_check, - 'url': url, - 'interactive': interactive}) + return apply_verification(verifier, + **{ + 'package_path': package_path, + 'package_check': package_check, + 'url': url, + 'interactive': interactive, + 'config': config, + }) def attempt_verification_per_domain(package_path, url): @@ -879,7 +883,7 @@ def get_integrity_file(package_path): return None -def check(url, download_path, interactive=True): +def check(url, download_path, config, interactive=True): """Run verification based on tar file url.""" package_name = filename_from_url(url) package_path = os.path.join(download_path, package_name) @@ -892,15 +896,15 @@ def check(url, download_path, interactive=True): print_info('Performing package integrity verification') verified = None if package_check is not None: - verified = from_disk(url, package_path, package_check, interactive=interactive) + verified = from_disk(url, package_path, package_check, config, interactive=interactive) elif package_path[-4:] == '.gem': signature_file = get_signature_file(url, download_path) - verified = from_disk(url, package_path, signature_file, interactive=interactive) + verified = from_disk(url, package_path, signature_file, config, interactive=interactive) else: print_info('None of {}.(asc|sig|sign|sha256) is found in {}'.format(package_name, download_path)) signature_file = get_signature_file(url, download_path) if signature_file is not None: - verified = from_disk(url, package_path, signature_file, interactive=interactive) + verified = from_disk(url, package_path, signature_file, config, interactive=interactive) if verified is None: print_info('Unable to find a signature') verified = attempt_verification_per_domain(package_path, url) @@ -936,7 +940,7 @@ def load_specfile(specfile): def main(args): """Verify tar content with signature.""" - from_disk(args.url, args.tar, args.sig) + from_disk(args.url, args.tar, args.sig, config.Config()) if __name__ == '__main__': diff --git a/autospec/pkg_scan.py b/autospec/pkg_scan.py index d232199..89f939a 100644 --- a/autospec/pkg_scan.py +++ b/autospec/pkg_scan.py @@ -17,11 +17,10 @@ # import subprocess -import config import util -def get_whatrequires(pkg): +def get_whatrequires(pkg, yum_conf): """ Write list of packages. @@ -30,7 +29,7 @@ def get_whatrequires(pkg): """ # clean up dnf cache to avoid 'no more mirrors repo' error try: - subprocess.check_output(['dnf', '--config', config.yum_conf, + subprocess.check_output(['dnf', '--config', yum_conf, '--releasever', 'clear', 'clean', 'all']) except subprocess.CalledProcessError as err: util.print_warning("Unable to clean dnf repo: {}, {}".format(pkg, err)) @@ -38,7 +37,7 @@ def get_whatrequires(pkg): try: out = subprocess.check_output(['dnf', 'repoquery', - '--config', config.yum_conf, + '--config', yum_conf, '--releasever', 'clear', '--archlist=src', '--recursive', '--queryformat=%{NAME}', '--whatrequires', pkg]).decode('utf-8') diff --git a/autospec/specdescription.py b/autospec/specdescription.py index 31eb72b..3420161 100644 --- a/autospec/specdescription.py +++ b/autospec/specdescription.py @@ -29,7 +29,6 @@ import os import re -import config import license import util @@ -84,7 +83,7 @@ def assign_description(description, score): default_description_score = score -def description_from_spec(specfile): +def description_from_spec(specfile, translations, blacklist): """Parse any existing RPM specfiles.""" try: with util.open_auto(specfile, 'r') as specfd: @@ -105,15 +104,15 @@ def description_from_spec(specfile): if line.startswith("License:") and not any(e in line for e in excludes): splits = line.split(":")[1:] words = ":".join(splits).strip() - if words in config.license_translations: + if words in translations: print("Adding license from spec:", words) - license.add_license(words) + license.add_license(words, translations, blacklist) else: words = clean_license_string(words).split() for word in words: if ":" not in word and not word.startswith("@"): print("Adding license from spec:", word) - license.add_license(word) + license.add_license(word, translations, blacklist) if line.startswith("Summary: "): assign_summary(line[9:], 4) @@ -128,7 +127,7 @@ def description_from_spec(specfile): assign_description(specdesc, 4) -def description_from_pkginfo(pkginfo): +def description_from_pkginfo(pkginfo, translations, blacklist): """Parse existing package info files.""" try: with util.open_auto(pkginfo, 'r') as pkgfd: @@ -146,15 +145,15 @@ def description_from_pkginfo(pkginfo): if line.lower().startswith("license:") and not any(e in line for e in excludes): splits = line.split(":")[1:] words = ":".join(splits).strip() - if words in config.license_translations: + if words in translations: print("Adding license from PKG-INFO:", words) - license.add_license(words) + license.add_license(words, translations, blacklist) else: words = clean_license_string(words).split() for word in words: if ":" not in word: print("Adding license from PKG-INFO:", word) - license.add_license(word) + license.add_license(word, translations, blacklist) for sub in ["Summary: ", "abstract: "]: if line.startswith(sub): @@ -241,7 +240,7 @@ def description_from_readme(readmefile): assign_description(desc, score) -def scan_for_description(package, dirn): +def scan_for_description(package, dirn, translations, blacklist): """Scan the project directory for things we can use to guess a description and summary.""" test_pat = re.compile(r"tests?") dirpath_seen = "" @@ -253,13 +252,13 @@ def scan_for_description(package, dirn): if name.lower().endswith(".pdf"): continue if name.lower().endswith(".spec"): - description_from_spec(os.path.join(dirpath, name)) + description_from_spec(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith("pkg-info"): - description_from_pkginfo(os.path.join(dirpath, name)) + description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith("meta.yml"): - description_from_pkginfo(os.path.join(dirpath, name)) + description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith("description"): - description_from_pkginfo(os.path.join(dirpath, name)) + description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist) if name.lower().endswith(".pc"): summary_from_pkgconfig(os.path.join(dirpath, name), package) if name.startswith("DESCRIPTION"): @@ -272,13 +271,13 @@ def scan_for_description(package, dirn): print("Summary :", default_summary.strip()) -def load_specfile(specfile): +def load_specfile(specfile, description, summary): """Load specfile with parse results.""" - if config.custom_desc: - specfile.default_desc = "\n".join(config.custom_desc) + if description: + specfile.default_desc = "\n".join(description) else: specfile.default_desc = default_description - if config.custom_summ: - specfile.default_sum = config.custom_summ[0] + if summary: + specfile.default_sum = summary[0] else: specfile.default_sum = default_summary diff --git a/autospec/specfiles.py b/autospec/specfiles.py index 4a96d47..5b42460 100644 --- a/autospec/specfiles.py +++ b/autospec/specfiles.py @@ -26,7 +26,6 @@ import types from collections import OrderedDict import buildreq -import config import tarball from util import _file_write from util import open_auto @@ -35,12 +34,13 @@ from util import open_auto class Specfile(object): """Holds data and methods needed to write the spec file.""" - def __init__(self, url, version, name, release): + def __init__(self, url, version, name, release, config): """Add default information for specfile template.""" self.url = url self.version = version self.name = name self.release = release + self.config = config self.keepstatic = False self.urlban = "" self.no_autostart = False @@ -179,9 +179,9 @@ class Specfile(object): self._write("Source{0} : {1}\n".format(count, source)) # if package is verified, include the signature in the source tarball - if self.keyid and config.signature: + if self.keyid and self.config.signature: count += 1 - self._write_strip(f"Source{count} : {config.signature}") + self._write_strip(f"Source{count} : {self.config.signature}") for source in self.extra_sources: count += 1 @@ -224,13 +224,13 @@ class Specfile(object): def write_strip_command(self): """Write commands to prevent stripping binary if requested.""" - if config.config_opts['nostrip']: + if self.config.config_opts['nostrip']: self._write("# Suppress stripping binaries\n") self._write("%define __strip /bin/true\n%define debug_package %{nil}\n") def write_debug_command(self): """Write commands to prevent debug info generation if requested.""" - if config.config_opts['nodebug']: + if self.config.config_opts['nodebug']: self._write("# Suppress generation of debuginfo\n") self._write("%global debug_package %{nil}\n") @@ -243,8 +243,8 @@ class Specfile(object): for count, patch in enumerate(self.patches): self._write("Patch{0}: {1}\n".format(count + 1, patch.split()[0])) # Write the version-specific patches - for version in config.verpatches: - for count, patch in enumerate(config.verpatches[version], start=count + 1): + for version in self.config.verpatches: + for count, patch in enumerate(self.config.verpatches[version], start=count + 1): self._write("Patch{0}: {1}\n".format(count + 1, patch.split()[0])) def write_description(self): @@ -270,9 +270,9 @@ class Specfile(object): deps["libexec"] = ["config", "license"] deps["lib32"] = ["data", "license"] deps["python"] = ["python3"] - if config.config_opts['dev_requires_extras']: + if self.config.config_opts.get('dev_requires_extras'): deps["dev"].append("extras") - if config.config_opts['openmpi']: + if self.config.config_opts.get('openmpi'): deps["dev"].append("openmpi") for k, v in self.custom_extras.items(): if "requires" in v: @@ -355,7 +355,7 @@ class Specfile(object): if pkg in ["ignore", "main", "locales"]: continue for script in ["post", "pre"]: - content = config.read_conf_file("{}.{}".format(script, pkg)) + content = self.config.read_conf_file("{}.{}".format(script, pkg)) if content: self._write("\n%{0} {1}\n".format(script, pkg)) content = ['{}\n'.format(l) for l in content] @@ -401,7 +401,7 @@ class Specfile(object): if export_epoch: # time.time() returns a float, but we only need second-precision self._write_strip("export SOURCE_DATE_EPOCH={}".format(int(time.time()))) - if config.config_opts['asneeded']: + if self.config.config_opts['asneeded']: self._write_strip("unset LD_AS_NEEDED\n") def write_proxy_exports(self): @@ -418,9 +418,9 @@ class Specfile(object): self._write_strip("{}\n".format(line)) self._write_strip("## make_prepend end") if build32: - self._write_strip("make {} {} {}".format(config.parallel_build, self.extra_make, self.extra32_make)) + self._write_strip("make {} {} {}".format(self.config.parallel_build, self.extra_make, self.extra32_make)) else: - self._write_strip("make {} {}".format(config.parallel_build, self.extra_make)) + self._write_strip("make {} {}".format(self.config.parallel_build, self.extra_make)) def write_install_openmpi(self): """Write make install line (openmpi) to spec file.""" @@ -533,19 +533,19 @@ class Specfile(object): destination)) self.apply_patches() if self.default_pattern != 'cmake': - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ..") self._write_strip("cp -a {} build32".format(self.tarball_prefix)) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ..") self._write_strip("cp -a {} buildavx2".format(self.tarball_prefix)) self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ..") self._write_strip("cp -a {} buildavx512".format(self.tarball_prefix)) self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd ..") self._write_strip("cp -a {} build-openmpi".format(self.tarball_prefix)) self._write_strip("popd") @@ -570,7 +570,7 @@ class Specfile(object): # a source of headaches for downstream users. self._write_strip("export GCC_IGNORE_WERROR=1\n") - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: self._write_strip("export CC=clang\n") self._write_strip("export CXX=clang++\n") self._write_strip("export LD=ld.gold\n") @@ -583,12 +583,12 @@ class Specfile(object): if not self.set_gopath: self._write_strip("export GOPROXY=file:///usr/share/goproxy") - if config.config_opts['optimize_size']: - if config.config_opts['use_clang']: + if self.config.config_opts['optimize_size']: + if self.config.config_opts['use_clang']: flags.extend(["-Os", "-ffunction-sections", "-fdata-sections"]) else: flags.extend(["-Os", "-ffunction-sections", "-fdata-sections", "-fno-semantic-interposition"]) - if config.config_opts['security_sensitive']: + if self.config.config_opts['security_sensitive']: flags.append("-fstack-protector-strong") if arch == 'x86_64': flags.append("-mzero-caller-saved-regs=used") @@ -596,10 +596,10 @@ class Specfile(object): flags.extend(["-O3", "-march=haswell"]) if self.need_avx512_flags: flags.extend(["-O3", "-march=skylake-avx512"]) - if config.config_opts['insecure_build']: + if self.config.config_opts['insecure_build']: self._write_strip('export CFLAGS="-O3 -g -fopt-info-vec "\n') self._write_strip("unset LDFLAGS\n") - if config.config_opts['conservative_flags']: + if self.config.config_opts['conservative_flags']: self._write_strip('export CFLAGS="-O2 -g -Wp,-D_FORTIFY_SOURCE=2 ' "-fexceptions -fstack-protector " "--param=ssp-buffer-size=32 -Wformat " @@ -608,17 +608,17 @@ class Specfile(object): '-march=westmere -mtune=haswell"\n') self._write_strip("export CXXFLAGS=$CFLAGS\n") self._write_strip("unset LDFLAGS\n") - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: self._write_strip("unset LDFLAGS\n") - if config.config_opts['funroll-loops']: - if config.config_opts['use_clang']: + if self.config.config_opts['funroll-loops']: + if self.config.config_opts['use_clang']: flags.extend(["-O3"]) else: flags.extend(["-O3", "-fno-semantic-interposition", "-falign-functions=32", "-fno-math-errno", "-fno-trapping-math"]) if self.default_pattern != 'qmake': - if config.config_opts['use_lto']: + if self.config.config_opts['use_lto']: flags.extend(["-O3", lto, "-ffat-lto-objects"]) - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: self._write_strip("export AR=llvm-ar\n") self._write_strip("export RANLIB=llvm-ranlib\n") self._write_strip("export NM=llvm-nm\n") @@ -628,26 +628,26 @@ class Specfile(object): self._write_strip("export NM=gcc-nm\n") else: flags.extend(["-fno-lto"]) - if config.config_opts['fast-math']: + if self.config.config_opts['fast-math']: flags.extend(["-ffast-math", "-ftree-loop-vectorize"]) - if config.config_opts['pgo']: + if self.config.config_opts['pgo']: flags.extend(["-O3"]) if self.gcov_file: flags = list(filter((lto).__ne__, flags)) flags.extend(["-O3", "-fauto-profile=%{{SOURCE{0}}}".format(self.source_index[self.sources["gcov"][0]])]) - if flags or config.config_opts['broken_c++']: + if flags or self.config.config_opts['broken_c++']: flags = sorted(list(set(flags))) self._write_strip('export CFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags))) self._write_strip('export FCFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags))) self._write_strip('export FFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags))) # leave the export CXXFLAGS line open in case self._write('export CXXFLAGS="$CXXFLAGS {0} '.format(" ".join(flags))) - if config.config_opts['broken_c++']: + if self.config.config_opts['broken_c++']: self._write('-std=gnu++98') # close the open quote from CXXFLAGS export and add newline self._write('"\n') - if config.profile_payload and config.profile_payload[0] and not self.need_avx2_flags: + if self.config.profile_payload and self.config.profile_payload[0] and not self.need_avx2_flags: genflags = [] useflags = [] genflags.extend(["-fprofile-generate", "-fprofile-dir=/var/tmp/pgo", "-fprofile-update=atomic"]) @@ -667,7 +667,7 @@ class Specfile(object): def write_check(self): """Write check section to spec file.""" - if self.tests_config and not config.config_opts['skip_tests']: + if self.tests_config and not self.config.config_opts['skip_tests']: self._write_strip("%check") self._write_strip("export LANG=C.UTF-8") self.write_proxy_exports() @@ -685,7 +685,7 @@ class Specfile(object): def write_profile_payload(self, pattern=None): """Write the profile_payload specified for this package.""" - if not config.profile_payload: + if not self.config.profile_payload: return use_subdir = True init = "" @@ -694,20 +694,20 @@ class Specfile(object): init = f"{self.get_profile_generate_flags()}" \ f"%configure " \ f"{self.disable_static} " \ - f"{config.extra_configure} " \ - f"{config.extra_configure64}" + f"{self.config.extra_configure} " \ + f"{self.config.extra_configure64}" elif pattern == "configure_ac": init = f"{self.get_profile_generate_flags()}" \ f"%reconfigure " \ f"{self.disable_static} " \ - f"{config.extra_configure} " \ - f"{config.extra_configure64}" + f"{self.config.extra_configure} " \ + f"{self.config.extra_configure64}" elif pattern == "autogen": init = f"{self.get_profile_generate_flags()}" \ f"%autogen " \ f"{self.disable_static} " \ - f"{config.extra_configure} " \ - f"{config.extra_configure64}" + f"{self.config.extra_configure} " \ + f"{self.config.extra_configure64}" elif pattern == "cmake": use_subdir = False init = f"{self.get_profile_generate_flags()}" @@ -723,7 +723,7 @@ class Specfile(object): if use_subdir and self.subdir: self._write_strip("popd") self._write_strip("\n") - self._write_strip("\n".join(config.profile_payload)) + self._write_strip("\n".join(self.config.profile_payload)) self._write_strip("\nmake clean\n") if post: self._write_strip(post) @@ -738,7 +738,7 @@ class Specfile(object): self.write_license_files() - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self._write_strip("%make_install32 {} {}".format(self.extra_make_install, self.extra_make32_install)) @@ -750,17 +750,17 @@ class Specfile(object): self._write_strip("fi") self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ../buildavx512/" + self.subdir) self._write_strip("%s_avx512 %s\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2/" + self.subdir) self._write_strip("%s_avx2 %s\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd ../build-openmpi/" + self.subdir) self.write_install_openmpi() self._write_strip("popd") @@ -1004,7 +1004,7 @@ class Specfile(object): if self.subdir: self._write_strip("pushd " + self.subdir) - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd clr-build32") self._write_strip("%make_install32 {} {}".format(self.extra_make_install, self.extra_make32_install)) @@ -1016,17 +1016,17 @@ class Specfile(object): self._write_strip("fi") self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd clr-build-avx512") self._write_strip("%s_avx512 %s || :\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd clr-build-avx2") self._write_strip("%s_avx2 %s || :\n" % (self.install_macro, self.extra_make_install)) self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd clr-build-openmpi") self.write_install_openmpi() self._write_strip("popd") @@ -1040,8 +1040,7 @@ class Specfile(object): self.write_find_lang() - @staticmethod - def get_profile_generate_flags(): + def get_profile_generate_flags(self): """Return profile generate flags if proper configuration is set. If config.profile_payload is non-empty, returns @@ -1053,7 +1052,7 @@ class Specfile(object): otherwise an empty string is returned. """ - if config.profile_payload and config.profile_payload[0]: + if self.config.profile_payload and self.config.profile_payload[0]: return 'CFLAGS="${CFLAGS_GENERATE}" ' \ 'CXXFLAGS="${CXXFLAGS_GENERATE}" ' \ 'FFLAGS="${FFLAGS_GENERATE}" ' \ @@ -1061,8 +1060,7 @@ class Specfile(object): 'LDFLAGS="${LDFLAGS_GENERATE}" ' return "" - @staticmethod - def get_profile_use_flags(): + def get_profile_use_flags(self): """Return profile generate flags if proper configuration is set. If config.profile_payload is non-empty, returns @@ -1074,7 +1072,7 @@ class Specfile(object): otherwise an empty string is returned. """ - if config.profile_payload and config.profile_payload[0]: + if self.config.profile_payload and self.config.profile_payload[0]: return 'CFLAGS="${CFLAGS_USE}" ' \ 'CXXFLAGS="${CXXFLAGS_USE}" ' \ 'FFLAGS="${FFLAGS_USE}" ' \ @@ -1118,13 +1116,13 @@ class Specfile(object): self._write_strip("{0}%configure {1} {2} {3}" .format(self.get_profile_use_flags(), self.disable_static, - config.extra_configure, - config.extra_configure64)) + self.config.extra_configure, + self.config.extra_configure64)) self.write_make_line() if self.subdir: self._write_strip("popd") self._write_strip("\n") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_build_prepend() self.write_32bit_exports() @@ -1134,12 +1132,12 @@ class Specfile(object): "--host=i686-generic-linux-gnu " "--target=i686-clr-linux-gnu" .format(self.disable_static, - config.extra_configure, - config.extra_configure32)) + self.config.extra_configure, + self.config.extra_configure32)) self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx2/" + self.subdir) self.write_build_prepend() @@ -1148,12 +1146,12 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self._write_strip("%configure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx2)) + self.config.extra_configure, + self.config.extra_configure_avx2)) self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx512/" + self.subdir) self.write_build_prepend() @@ -1162,12 +1160,12 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=skylake-avx512\"") self._write_strip("%configure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx512)) + self.config.extra_configure, + self.config.extra_configure_avx512)) self.write_make_line() self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("pushd ../build-openmpi/" + self.subdir) self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh") self._write_strip("module load openmpi") @@ -1178,9 +1176,9 @@ class Specfile(object): self._write_strip("export FFLAGS=\"$FFLAGS -m64 -march=haswell\"") self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self._write_strip("./configure {0} \\\n{1} {2}" - .format(config.conf_args_openmpi, + .format(self.config.conf_args_openmpi, self.disable_static, - config.extra_configure_openmpi)) + self.config.extra_configure_openmpi)) self.write_make_line() self._write_strip("module unload openmpi") self._write_strip("popd") @@ -1199,12 +1197,12 @@ class Specfile(object): self._write_strip("{0}%reconfigure {1} {2} {3}" .format(self.get_profile_use_flags(), self.disable_static, - config.extra_configure, - config.extra_configure64)) + self.config.extra_configure, + self.config.extra_configure64)) self.write_make_line() if self.subdir: self._write_strip("popd") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_build_prepend() self.write_32bit_exports() @@ -1214,12 +1212,12 @@ class Specfile(object): "--host=i686-generic-linux-gnu " "--target=i686-clr-linux-gnu" .format(self.disable_static, - config.extra_configure, - config.extra_configure32)) + self.config.extra_configure, + self.config.extra_configure32)) self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx2/" + self.subdir) self.write_build_prepend() @@ -1228,12 +1226,12 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self._write_strip("%reconfigure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx2)) + self.config.extra_configure, + self.config.extra_configure_avx2)) self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("pushd ../buildavx512/" + self.subdir) self.write_build_prepend() @@ -1242,8 +1240,8 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=skylake-avx512\"") self._write_strip("%reconfigure {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx512)) + self.config.extra_configure, + self.config.extra_configure_avx512)) self.write_make_line() self._write_strip("popd") @@ -1263,12 +1261,12 @@ class Specfile(object): if self.subdir: self._write_strip("popd") self._write_strip("\n") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_32bit_exports() self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2" + self.subdir) self.write_build_prepend() self._write_strip("export CFLAGS=\"$CFLAGS -m64 -march=haswell\"") @@ -1276,7 +1274,7 @@ class Specfile(object): self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"") self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ../buildavx512" + self.subdir) self.write_build_prepend() self._write_strip("export CFLAGS=\"$CFLAGS -m64 -march=skylake-avx512 -mprefer-vector-width=512\"") @@ -1298,11 +1296,11 @@ class Specfile(object): self._write_strip("{0}%autogen {1} {2} {3}" .format(self.get_profile_use_flags(), self.disable_static, - config.extra_configure, - config.extra_configure64)) + self.config.extra_configure, + self.config.extra_configure64)) self.write_make_line() self._write_strip("\n") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_build_prepend() self.write_32bit_exports() @@ -1312,12 +1310,12 @@ class Specfile(object): "--host=i686-generic-linux-gnu " "--target=i686-clr-linux-gnu" .format(self.disable_static, - config.extra_configure, - config.extra_configure32)) + self.config.extra_configure, + self.config.extra_configure32)) self.write_make_line(True) self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2/" + self.subdir) self.write_build_prepend() self._write_strip('export CFLAGS="$CFLAGS -m64 -march=haswell "') @@ -1325,12 +1323,12 @@ class Specfile(object): self._write_strip('export LDFLAGS="$LDFLAGS -m64 -march=haswell "') self._write_strip("%autogen {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx2)) + self.config.extra_configure, + self.config.extra_configure_avx2)) self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("pushd ../buildavx512/" + self.subdir) self.write_build_prepend() self._write_strip('export CFLAGS="$CFLAGS -m64 -march=skylake-avx512 "') @@ -1338,8 +1336,8 @@ class Specfile(object): self._write_strip('export LDFLAGS="$LDFLAGS -m64 -march=skylake-avx512 "') self._write_strip("%autogen {0} {1} {2} " .format(self.disable_static, - config.extra_configure, - config.extra_configure_avx512)) + self.config.extra_configure, + self.config.extra_configure_avx512)) self.write_make_line() self._write_strip("popd") @@ -1354,9 +1352,9 @@ class Specfile(object): self._write_strip("export MAKEFLAGS=%{?_smp_mflags}") if self.subdir: self._write_strip("pushd " + self.subdir) - self._write_strip("python3 setup.py build " + config.extra_configure) + self._write_strip("python3 setup.py build " + self.config.extra_configure) self._write_strip("\n") - if self.tests_config and not config.config_opts['skip_tests']: + if self.tests_config and not self.config.config_opts['skip_tests']: self._write_strip("%check") # Prevent setuptools from hitting the internet self.write_proxy_exports() @@ -1388,9 +1386,9 @@ class Specfile(object): self.write_variables() if self.subdir: self._write_strip("pushd " + self.subdir) - self._write_strip("python3.6 setup.py build -b py3 " + config.extra_configure) + self._write_strip("python3.6 setup.py build -b py3 " + self.config.extra_configure) self._write_strip("\n") - if self.tests_config and not config.config_opts['skip_tests']: + if self.tests_config and not self.config.config_opts['skip_tests']: self._write_strip("%check") # Prevent setuptools from hitting the internet self.write_proxy_exports() @@ -1535,7 +1533,7 @@ class Specfile(object): self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("mkdir -p clr-build-avx2") self._write_strip("pushd clr-build-avx2") saved_avx2flags = self.need_avx2_flags @@ -1549,7 +1547,7 @@ class Specfile(object): self.write_make_line() self._write_strip("popd") - if config.config_opts['use_avx512']: + if self.config.config_opts['use_avx512']: self._write_strip("mkdir -p clr-build-avx512") self._write_strip("pushd clr-build-avx512") saved_avx512flags = self.need_avx512_flags @@ -1563,7 +1561,7 @@ class Specfile(object): self.write_make_line() self._write_strip("popd") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("mkdir -p clr-build32") self._write_strip("pushd clr-build32") self.write_build_prepend() @@ -1577,7 +1575,7 @@ class Specfile(object): self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("popd") - if config.config_opts['openmpi']: + if self.config.config_opts['openmpi']: self._write_strip("mkdir -p clr-build-openmpi") self._write_strip("pushd clr-build-openmpi") self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh") @@ -1607,9 +1605,9 @@ class Specfile(object): def write_qmake_pattern(self): """Write qmake build pattern to spec file.""" extra_qmake_args = "" - if config.config_opts['use_clang']: + if self.config.config_opts['use_clang']: extra_qmake_args = "-spec linux-clang " - if config.config_opts['use_lto']: + if self.config.config_opts['use_lto']: extra_qmake_args += "-config ltcg -config fat-static-lto " else: extra_qmake_args += "QMAKE_CFLAGS+=-fno-lto QMAKE_CXXFLAGS+=-fno-lto " @@ -1624,18 +1622,18 @@ class Specfile(object): if self.subdir: self._write_strip("pushd " + self.subdir) - self._write_strip("%qmake {} {}".format(extra_qmake_args, config.extra_configure)) + self._write_strip("%qmake {} {}".format(extra_qmake_args, self.config.extra_configure)) self._write_strip("test -r config.log && cat config.log") self.write_make_line() if self.subdir: self._write_strip("popd") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip("pushd ../buildavx2/" + self.subdir) self._write("%qmake 'QT_CPU_FEATURES.x86_64 += avx avx2 bmi bmi2 f16c fma lzcnt popcnt'\\\n") self._write(" QMAKE_CFLAGS+=-march=haswell QMAKE_CXXFLAGS+=-march=haswell \\\n") - self._write(" QMAKE_LFLAGS+=-march=haswell {} {}\n".format(extra_qmake_args, config.extra_configure)) + self._write(" QMAKE_LFLAGS+=-march=haswell {} {}\n".format(extra_qmake_args, self.config.extra_configure)) self.write_make_line() self._write_strip("popd") @@ -1708,7 +1706,7 @@ class Specfile(object): self.write_proxy_exports() self._write_strip("export LANG=C.UTF-8") self.write_variables() - self._write_strip("scons{} {}".format(config.parallel_build, config.extra_configure)) + self._write_strip("scons{} {}".format(self.config.parallel_build, self.config.extra_configure)) self.write_build_append() self._write_strip("\n") self._write_strip("%install") @@ -1853,23 +1851,23 @@ class Specfile(object): if self.subdir: self._write_strip("pushd " + self.subdir) self._write_strip('CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" LDFLAGS="$LDFLAGS" meson --libdir=lib64 --prefix=/usr --buildtype=plain {0} {1} builddir' - .format(config.extra_configure, - config.extra_configure64)) + .format(self.config.extra_configure, + self.config.extra_configure64)) self._write_strip("ninja -v -C builddir") - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip('CFLAGS="$CFLAGS -m64 -march=haswell" CXXFLAGS="$CXXFLAGS -m64 -march=haswell " LDFLAGS="$LDFLAGS -m64 -march=haswell" ' 'meson --libdir=lib64/haswell --prefix=/usr --buildtype=plain {0} ' - '{1} builddiravx2'.format(config.extra_configure, config.extra_configure64)) + '{1} builddiravx2'.format(self.config.extra_configure, self.config.extra_configure64)) self._write_strip('ninja -v -C builddiravx2') if self.subdir: self._write_strip("popd") - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip("pushd ../build32/" + self.subdir) self.write_32bit_exports() self._write_strip('meson ' '--libdir=lib32 --prefix=/usr --buildtype=plain {0} {1} builddir' - .format(config.extra_configure, - config.extra_configure32)) + .format(self.config.extra_configure, + self.config.extra_configure32)) self._write_strip('ninja -v -C builddir') self._write_strip('popd') @@ -1879,7 +1877,7 @@ class Specfile(object): self._write_strip("%install") self.write_install_prepend() self.write_license_files() - if config.config_opts['32bit']: + if self.config.config_opts['32bit']: self._write_strip('pushd ../build32/' + self.subdir) self._write_strip('DESTDIR=%{buildroot} ninja -C builddir install') self._write_strip("if [ -d %{buildroot}/usr/lib32/pkgconfig ]") @@ -1891,7 +1889,7 @@ class Specfile(object): self._write_strip("popd") if self.subdir: self._write_strip("pushd " + self.subdir) - if config.config_opts['use_avx2']: + if self.config.config_opts['use_avx2']: self._write_strip('DESTDIR=%{buildroot} ninja -C builddiravx2 install') self._write_strip("DESTDIR=%{buildroot} ninja -C builddir install") @@ -1951,7 +1949,7 @@ class Specfile(object): # Write version-specific patch commands for version in self.verpatches: if self.verpatches[version]: - self._write("cd ../{}\n".format(self.build_dirs[config.versions[version]])) + self._write("cd ../{}\n".format(self.build_dirs[self.config.versions[version]])) for p in self.verpatches[version]: name = p.split(None, 1)[0] if name == p: diff --git a/autospec/tarball.py b/autospec/tarball.py index 075d077..b426314 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -27,7 +27,6 @@ from collections import OrderedDict import build import buildpattern import buildreq -import config import download from util import call, do_regex, get_sha1sum, print_fatal, write_out @@ -266,7 +265,7 @@ def detect_build_from_url(url): buildpattern.set_build_pattern("phpize", 10) -def set_multi_version(ver): +def set_multi_version(ver, config): """Add ver to multi_version set and return latest version.""" global multi_version @@ -287,7 +286,7 @@ def set_multi_version(ver): return latest -def name_and_version(name_arg, version_arg, filemanager): +def name_and_version(name_arg, version_arg, filemanager, config): """Parse the url for the package name and version.""" global rawname global url @@ -307,7 +306,7 @@ def name_and_version(name_arg, version_arg, filemanager): # rawname == name in this case name = name_arg rawname = name_arg - version = set_multi_version(version_arg) + version = set_multi_version(version_arg, config) return name, rawname, convert_version(version, name) name = name_arg @@ -439,7 +438,7 @@ def name_and_version(name_arg, version_arg, filemanager): # override name and version from commandline name = name_arg if name_arg else name version = version_arg if version_arg else version - version = set_multi_version(version) + version = set_multi_version(version, config) return name, rawname, convert_version(version, name) @@ -474,7 +473,7 @@ def process_go_archives(go_archives): buildpattern.sources["godep"] += [url_info, url_mod, url_zip] -def process_multiver_archives(main_src, multiver_archives): +def process_multiver_archives(main_src, multiver_archives, config): """Set up multiversion archives.""" config_versions = config.parse_config_versions(build.download_path) # Check if exist more than one version. @@ -485,10 +484,10 @@ def process_multiver_archives(main_src, multiver_archives): buildpattern.sources["version"].append(extraurl) multiver_archives.append(extraurl) multiver_archives.append('') - set_multi_version(None) + set_multi_version(None, config) -def process_archives(main_src, archives): +def process_archives(main_src, archives, config): """Process extra sources needed by package. This sources include: archives, go archives and multiversion. @@ -502,7 +501,7 @@ def process_archives(main_src, archives): process_go_archives(go_archives) # Add multiversion for the rest of the patterns else: - process_multiver_archives(main_src, multiver_archives) + process_multiver_archives(main_src, multiver_archives, config) full_archives = archives + go_archives + multiver_archives # Download and extract full list @@ -534,7 +533,7 @@ def extract_sources(main_src, archives_src): src.extract() -def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): +def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager, config): """Download and process the tarball at url_arg.""" global url global name @@ -552,7 +551,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): # Create the download path for content and set build.download_path create_download_path(target) # determine name and version of package - name, rawname, version = name_and_version(name_arg, ver_arg, filemanager) + name, rawname, version = name_and_version(name_arg, ver_arg, filemanager, config) # Store the top-level version config.versions[version] = url # set gcov file information, must be done after name is set since the gcov @@ -570,7 +569,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): print_header() # Download and process extra sources: archives, go archives and # multiversion - archives_src = process_archives(main_src, archives) + archives_src = process_archives(main_src, archives, config) # Extract all sources extract_sources(main_src, archives_src) diff --git a/tests/test_build.py b/tests/test_build.py index 8016152..d466afd 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,18 +3,12 @@ import tempfile import os from unittest.mock import patch, mock_open, MagicMock import build +import config import files class TestBuildpattern(unittest.TestCase): - @classmethod - def setUpClass(self): - """ - Class setup method to configure necessary modules - """ - build.config.setup_patterns() - def setUp(self): """ Test setup method to reset the buildpattern module @@ -25,7 +19,6 @@ class TestBuildpattern(unittest.TestCase): build.base_path = None build.download_path = None build.buildreq.buildreqs = set() - build.config.config_opts['32bit'] = False def test_setup_workingdir(self): """ @@ -40,10 +33,10 @@ class TestBuildpattern(unittest.TestCase): """ Test simple_pattern_pkgconfig with match """ - build.buildreq.config.config_opts['32bit'] = False build.simple_pattern_pkgconfig('line to test for testpkg.xyz', r'testpkg.xyz', - 'testpkg') + 'testpkg', + False) self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs) self.assertEqual(build.must_restart, 1) @@ -51,10 +44,10 @@ class TestBuildpattern(unittest.TestCase): """ Test simple_pattern_pkgconfig with match and 32bit option set """ - build.buildreq.config.config_opts['32bit'] = True build.simple_pattern_pkgconfig('line to test for testpkg.zyx', r'testpkg.zyx', - 'testpkgz') + 'testpkgz', + True) self.assertIn('pkgconfig(32testpkgz)', build.buildreq.buildreqs) self.assertIn('pkgconfig(testpkgz)', build.buildreq.buildreqs) self.assertEqual(build.must_restart, 1) @@ -65,7 +58,8 @@ class TestBuildpattern(unittest.TestCase): """ build.simple_pattern_pkgconfig('line to test for somepkg.xyz', r'testpkg.xyz', - 'testpkg') + 'testpkg', + False) self.assertEqual(build.buildreq.buildreqs, set()) self.assertEqual(build.must_restart, 0) @@ -95,7 +89,8 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with no match """ - build.failed_pattern('line to test for failure: somepkg', r'(test)', 0) + conf = config.Config() + build.failed_pattern('line to test for failure: somepkg', conf, r'(test)', 0) self.assertEqual(build.buildreq.buildreqs, set()) self.assertEqual(build.must_restart, 0) @@ -104,7 +99,8 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool unset and initial match, but no match in failed_commands. """ - build.failed_pattern('line to test for failure: testpkg', r'(test)', 0) + conf = config.Config() + build.failed_pattern('line to test for failure: testpkg', conf, r'(test)', 0) self.assertEqual(build.buildreq.buildreqs, set()) self.assertEqual(build.must_restart, 0) @@ -112,7 +108,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool unset and match in failed_commands """ - build.failed_pattern('line to test for failure: lex', r'(lex)', 0) + conf = config.Config() + conf.setup_patterns() + build.failed_pattern('line to test for failure: lex', conf, r'(lex)', 0) self.assertIn('flex', build.buildreq.buildreqs) self.assertEqual(build.must_restart, 1) @@ -120,8 +118,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to pkgconfig """ - build.buildreq.config.config_opts['32bit'] = False + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.xyz', + conf, r'(testpkg)', 0, # verbose=0 buildtool='pkgconfig') @@ -132,7 +131,10 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to R """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: testpkg.r', + conf, r'(testpkg)', 0, # verbose=0 buildtool='R') @@ -144,7 +146,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to perl """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.pl', + conf, r'(testpkg)', 0, # verbose=0 buildtool='perl') @@ -155,7 +159,9 @@ class TestBuildpattern(unittest.TestCase): """ Test failed_pattern with buildtool set to pypi """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.py', + conf, r'(testpkg)', 0, # verbose=0 buildtool='pypi') @@ -167,7 +173,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to ruby, but no match in config.gems, it should just prepend 'rubygem-' to the package name. """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg.rb', + conf, r'(testpkg)', 0, # verbose=0 buildtool='ruby') @@ -180,7 +188,10 @@ class TestBuildpattern(unittest.TestCase): config.gems. In the particular case of test/unit, the result should be rubygem-test-unit. """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: test/unit', + conf, r'(test/unit)', 0, # verbose=0 buildtool='ruby') @@ -192,7 +203,10 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to ruby table and a match in config.gems """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: test/unit', + conf, r'(test/unit)', 0, # verbose=0 buildtool='ruby table') @@ -204,7 +218,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to ruby table but no match in config.gems. This should not modify anything. """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg', + conf, r'(testpkg)', 0, # verbose=0 buildtool='ruby table') @@ -216,7 +232,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to maven, but no match in config.maven_jars, it should just prepend 'mvn-' to the package name. """ + conf = config.Config() build.failed_pattern('line to test for failure: testpkg', + conf, r'(testpkg)', 0, # verbose=0 buildtool='maven') @@ -229,7 +247,10 @@ class TestBuildpattern(unittest.TestCase): config.maven_jars. In the particular case of aether, the corresponding maven jar is 'mvn-aether-core' """ + conf = config.Config() + conf.setup_patterns() build.failed_pattern('line to test for failure: aether', + conf, r'(aether)', 0, # verbose=0 buildtool='maven') @@ -244,8 +265,6 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -269,8 +288,6 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -295,8 +312,6 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -320,18 +335,19 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() - build.config.config_opts['32bit'] = True + conf = config.Config() + conf.setup_patterns() + conf.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\nwhich: no qmake\nexiting' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup @@ -347,17 +363,18 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\nchecking for Apache test module support\nexiting' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup @@ -369,11 +386,12 @@ class TestBuildpattern(unittest.TestCase): Test parse_build_results with a test log indicating failure due to a missing package. """ - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call open_auto_backup = build.util.open_auto build.util.call = MagicMock(return_value=None) - fm = files.FileManager() + fm = files.FileManager(conf) with open('tests/builderrors', 'r') as f: builderrors = f.readlines() @@ -382,7 +400,7 @@ class TestBuildpattern(unittest.TestCase): input, output = error.strip('\n').split('|') build.buildreq.buildreqs = set() build.util.open_auto = mock_open(read_data=input) - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) self.assertIn(output, build.buildreq.buildreqs) self.assertGreater(build.must_restart, 0) @@ -398,10 +416,11 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\n' \ @@ -414,7 +433,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup @@ -432,10 +451,11 @@ class TestBuildpattern(unittest.TestCase): def mock_util_call(cmd): del cmd - build.config.setup_patterns() + conf = config.Config() + conf.setup_patterns() call_backup = build.util.call build.util.call = mock_util_call - fm = files.FileManager() + fm = files.FileManager(conf) open_name = 'build.util.open_auto' content = 'line 1\n' \ @@ -450,7 +470,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm) + build.parse_build_results('testname', 0, fm, conf) build.util.call = call_backup diff --git a/tests/test_buildreq.py b/tests/test_buildreq.py index d78298e..09d23e3 100644 --- a/tests/test_buildreq.py +++ b/tests/test_buildreq.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, mock_open, patch import json import io import buildreq +import config class TestBuildreq(unittest.TestCase): @@ -15,7 +16,6 @@ class TestBuildreq(unittest.TestCase): Class setup method to configure necessary modules """ buildreq.banned_buildreqs.add('bannedreq') - buildreq.config.setup_patterns() def setUp(self): """ @@ -26,9 +26,6 @@ class TestBuildreq(unittest.TestCase): buildreq.requires = set() buildreq.verbose = False buildreq.cargo_bin = False - buildreq.config.config_opts['32bit'] = False - buildreq.config.cmake_modules = {} - buildreq.config.os_packages = set() buildreq.buildpattern.pattern_strength = 0 def test_add_buildreq(self): @@ -53,29 +50,28 @@ class TestBuildreq(unittest.TestCase): buildreqs but not yet present in requires """ buildreq.add_buildreq('testreq') - self.assertTrue(buildreq.add_requires('testreq')) + self.assertTrue(buildreq.add_requires('testreq', ['testreq'])) self.assertIn('testreq', buildreq.requires) def test_add_requires_not_in_buildreqs(self): """ Test add_requires with unbanned new req not present in buildreqs. """ - self.assertFalse(buildreq.add_requires('testreq')) + self.assertFalse(buildreq.add_requires('testreq', [])) self.assertNotIn('testreq', buildreq.requires) def test_add_pkgconfig_buildreq(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to False """ - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq')) + self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', False)) self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) def test_add_pkgconfig_buildreq_32bit(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to True """ - buildreq.config.config_opts['32bit'] = True - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq')) + self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', True)) self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) self.assertIn('pkgconfig(32testreq)', buildreq.buildreqs) @@ -83,14 +79,14 @@ class TestBuildreq(unittest.TestCase): """ Test configure_ac_line with standard pattern """ - buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])') + buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False) self.assertIn('ncurses-devel', buildreq.buildreqs) def test_configure_ac_line_comment(self): """ Test configure_ac_line with commented line """ - buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])') + buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False) self.assertEqual(buildreq.buildreqs, set()) def test_configure_ac_line_pkg_check_modules(self): @@ -101,7 +97,7 @@ class TestBuildreq(unittest.TestCase): buildreq.configure_ac_line( 'PKG_CHECK_MODULES(prefix, ' '[module > 2 module2 < 2], ' - 'action-if-found, action-if-not-found)') + 'action-if-found, action-if-not-found)', False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) @@ -112,7 +108,7 @@ class TestBuildreq(unittest.TestCase): buildreq.configure_ac_line( 'XDT_CHECK_PACKAGE(prefix, ' '[module = 2 module2 > 9], ' - 'action-if-found, action-if-not-found)') + 'action-if-found, action-if-not-found)', False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) @@ -122,7 +118,7 @@ class TestBuildreq(unittest.TestCase): """ buildreq.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], ' 'action-if-found, ' - 'action-if-not-found)') + 'action-if-not-found)', False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(module1)', 'pkgconfig(module2)'])) @@ -143,7 +139,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_configure_ac(os.path.join(tmpd, 'fname')) + buildreq.parse_configure_ac(os.path.join(tmpd, 'fname'), False) self.assertEqual(buildreq.buildpattern.default_pattern, 'configure_ac') self.assertEqual(buildreq.buildreqs, @@ -205,7 +201,7 @@ class TestBuildreq(unittest.TestCase): content = 'does not matter, let us mock' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.parse_cargo_toml('filename') + buildreq.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3']) buildreq.os.path.exists = exists_backup buildreq.toml.loads = loads_backup @@ -249,11 +245,13 @@ class TestBuildreq(unittest.TestCase): """ Test rakefile parsing with both configured gems and unconfigured gems """ + conf = config.Config() + conf.setup_patterns() open_name = 'buildreq.util.open_auto' content = "line1\nrequire 'bundler/gem_tasks'\nline3\nrequire 'nope'" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.rakefile('filename') + buildreq.rakefile('filename', conf.gems) self.assertEqual(buildreq.buildreqs, set(['rubygem-rubygems-tasks'])) @@ -291,7 +289,7 @@ class TestBuildreq(unittest.TestCase): 'req7 == 3.3.3\n' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename') + buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) @@ -307,7 +305,7 @@ class TestBuildreq(unittest.TestCase): ' req7 == 3.3.3\n ' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename') + buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) @@ -321,7 +319,7 @@ class TestBuildreq(unittest.TestCase): "setup_requires=['req2']" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) self.assertEqual(buildreq.requires, set(['req1'])) @@ -336,7 +334,7 @@ class TestBuildreq(unittest.TestCase): "'req3']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) @@ -354,7 +352,7 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) @@ -372,7 +370,7 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) self.assertEqual(buildreq.requires, set(['req1', 'req2'])) @@ -385,7 +383,7 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=[reqname, 'req1', 'req2']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', ['req1', 'req2']) self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) self.assertEqual(buildreq.requires, set(['req1', 'req2'])) @@ -398,7 +396,7 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=reqname" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename') + buildreq.add_setup_py_requires('filename', []) self.assertEqual(buildreq.buildreqs, set()) self.assertEqual(buildreq.requires, set()) @@ -425,6 +423,7 @@ class TestBuildreq(unittest.TestCase): much to test here that uses the same logic, a representative test should be sufficient. """ + conf = config.Config() with tempfile.TemporaryDirectory() as tmpd: os.mkdir(os.path.join(tmpd, 'subdir')) open(os.path.join(tmpd, 'subdir', 'test.go'), 'w').close() @@ -433,7 +432,7 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'SConstruct'), 'w').close() open(os.path.join(tmpd, 'meson.build'), 'w').close() - buildreq.scan_for_configure(tmpd, "", "") + buildreq.scan_for_configure(tmpd, "", "", conf) self.assertEqual(buildreq.buildreqs, set(['buildreq-golang', 'buildreq-cmake', 'buildreq-scons', 'buildreq-distutils3', 'buildreq-meson'])) @@ -443,6 +442,7 @@ class TestBuildreq(unittest.TestCase): Test scan_for_configure when distutils is being used for the build pattern to test pypi metadata handling. """ + conf = config.Config() orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score orig_pypi_name = buildreq.pypidata.get_pypi_name @@ -459,7 +459,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: os.mkdir(os.path.join(tmpd, 'subdir')) open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd) + buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary @@ -476,6 +476,7 @@ class TestBuildreq(unittest.TestCase): Test scan_for_configure when distutils is being used for the build pattern to test pypi metadata file override handling. """ + conf = config.Config() open_name = 'buildreq.open' orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score @@ -491,7 +492,7 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() open(os.path.join(tmpd, 'pypi.json'), 'w').close() with patch(open_name, m_open, create=True): - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd) + buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary @@ -506,11 +507,13 @@ class TestBuildreq(unittest.TestCase): Test parse_cmake to ensure accurate detection of versioned and unversioned pkgconfig modules. """ + conf = config.Config() + conf.setup_patterns() content = 'pkg_check_modules(GLIB gio-unix-2.0>=2.46.0 glib-2.0 REQUIRED)' with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) @@ -520,11 +523,13 @@ class TestBuildreq(unittest.TestCase): Test parse_cmake to ensure accurate handling of versioned pkgconfig modules with whitespace. """ + conf = config.Config() + conf.setup_patterns() content = 'pkg_check_modules(GLIB gio-unix-2.0 >= 2.46.0 glib-2.0 REQUIRED)' with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) @@ -533,6 +538,8 @@ class TestBuildreq(unittest.TestCase): """ Test parse_cmake to ensure it ignores pkg_check_modules in comments. """ + conf = config.Config() + conf.setup_patterns() content = ''' # For example, consider the following patch to some CMakeLists.txt. # - pkg_check_modules(FOO REQUIRED foo>=1.0) @@ -541,7 +548,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set([])) @@ -551,11 +558,13 @@ class TestBuildreq(unittest.TestCase): Test parse_cmake to ensure accurate handling of versioned pkgconfig modules with variable version strings. """ + conf = config.Config() + conf.setup_patterns() content = 'pkg_check_modules(AVCODEC libavcodec${_avcodec_ver} libavutil$_avutil_ver)' with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)'])) @@ -564,7 +573,7 @@ class TestBuildreq(unittest.TestCase): """ Test parse_cmake to ensure accurate handling of find_package. """ - buildreq.config.cmake_modules = { + cmake_modules = { "valid": "valid", "valid_but_commented": "valid_but_commented", "different_name": "another_name", @@ -578,7 +587,7 @@ find_package(different_name) with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname')) + buildreq.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False) self.assertEqual(buildreq.buildreqs, set(['valid', 'another_name'])) @@ -679,44 +688,37 @@ find_package(different_name) result = buildreq._get_desc_field("Field2", "\n".join(lines)) self.assertEqual(result, []) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_depends(self, os_pkgs): + def test_parse_r_desc_depends(self): """Test parsing of R description Depends field.""" pkgs = ['R-pkg1'] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = 'Depends: pkg1' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg1' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_imports(self, os_pkgs): + def test_parse_r_desc_imports(self): """Test parsing of an R description Imports field.""" pkgs = ['R-pkg2'] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = 'Imports: pkg2' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg2' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_linkingto(self, os_pkgs): + def test_parse_r_desc_linkingto(self): """Test parsing of an R description LinkingTo field.""" pkgs = ['R-pkg3'] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = 'LinkingTo: pkg3' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg3' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_multiple(self, os_pkgs): + def test_parse_r_desc_multiple(self): """Test parsing of an R description file that captures multiple fields.""" pkgs = [ 'R-pkg1', @@ -724,7 +726,6 @@ find_package(different_name) 'R-pkg3', 'R-pkg4', ] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = [ 'Field1: foo', @@ -735,7 +736,7 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertFalse('R-foo' in buildreq.buildreqs) self.assertFalse('R-bar' in buildreq.buildreqs) self.assertTrue('R-pkg1' in buildreq.buildreqs) @@ -743,13 +744,11 @@ find_package(different_name) self.assertTrue('R-pkg3' in buildreq.buildreqs) self.assertTrue('R-pkg4' in buildreq.buildreqs) - @patch('buildreq.config.os_packages') - def test_parse_r_desc_not_in_os(self, os_pkgs): + def test_parse_r_desc_not_in_os(self): """Test parsing of an R description file with some non-OS packages.""" pkgs = [ 'R-pkg1', ] - os_pkgs.__contains__.side_effect = lambda val: val in pkgs open_name = 'buildreq.util.open_auto' content = [ 'Imports: pkg1, pkg2', @@ -757,7 +756,7 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename') + buildreq.parse_r_description('filename', pkgs) self.assertTrue('R-pkg1' in buildreq.buildreqs) self.assertFalse('R-pkg2' in buildreq.buildreqs) self.assertFalse('R-pkg3' in buildreq.buildreqs) diff --git a/tests/test_check.py b/tests/test_check.py index aaebfec..f8925f6 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import mock_open, patch import check - +import config def mock_generator(rv=None): def mock_f(*args, **kwargs): @@ -19,12 +19,6 @@ class TestTest(unittest.TestCase): @classmethod def setUpClass(self): self.open_name = 'check.util.open_auto' - check.config.config_opts['skip_tests'] = False - check.config.config_opts['allow_test_failures'] = False - check.config.config_opts['32bit'] = False - check.config.config_opts['use_avx2'] = False - check.config.config_opts['use_avx512'] = False - check.config.config_opts['openmpi'] = False check.os.path.isfile = mock_generator(True) @classmethod @@ -49,7 +43,7 @@ class TestTest(unittest.TestCase): m_open = mock_open() open_name = 'util.open' with patch(open_name, m_open, create=True): - check.check_regression('pkgdir') + check.check_regression('pkgdir', False) check.count.parse_log = parse_log_backup @@ -72,7 +66,7 @@ class TestTest(unittest.TestCase): m_open = mock_open() open_name = 'util.open' with patch(open_name, m_open, create=True): - check.check_regression('pkgdir') + check.check_regression('pkgdir', False) check.count.parse_log = parse_log_backup @@ -94,13 +88,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) content = 'check:' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -111,12 +106,13 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite via Makefile.am """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.am']) m_open = mock_open() with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure_ac" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -127,10 +123,11 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.PL']) check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test') @@ -139,13 +136,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite via Makefile.in """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) content = 'test:' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -155,13 +153,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with setup.py suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['setup.py']) content = 'test_suite' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "distutils3" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -173,13 +172,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with cmake suite """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['CMakeLists.txt']) content = 'enable_testing' m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cmake" - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -191,9 +191,10 @@ class TestTest(unittest.TestCase): Test scan_for_tests with tox.ini in the files list, should add several build requirements """ + conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['tox.ini']) - check.scan_for_tests('pkgdir') + check.scan_for_tests('pkgdir', conf) check.os.listdir = listdir_backup self.assertEqual(check.buildreq.buildreqs, set(['tox', diff --git a/tests/test_commitmessage.py b/tests/test_commitmessage.py index b4b3b96..4256189 100644 --- a/tests/test_commitmessage.py +++ b/tests/test_commitmessage.py @@ -3,6 +3,7 @@ import unittest.mock as mock import os import tempfile import commitmessage +import config class TestCommitmessage(unittest.TestCase): @@ -10,7 +11,6 @@ class TestCommitmessage(unittest.TestCase): def setUp(self): commitmessage.tarball.name = 'testball' commitmessage.tarball.version = '0.0.1' - commitmessage.config.old_version = '0.0.0' self.workingdir = tempfile.TemporaryDirectory() commitmessage.build.setup_workingdir(self.workingdir.name) @@ -59,7 +59,7 @@ class TestCommitmessage(unittest.TestCase): # last items expected_msg = [""] + GOOD_NEWS.split('\n')[3:13] expected_cvs = set() - self.assertEqual(commitmessage.process_NEWS('NEWS'), + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), (expected_msg, expected_cvs)) def test_process_NEWS_bad_news(self): @@ -71,7 +71,7 @@ class TestCommitmessage(unittest.TestCase): with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile: # make GOOD_NEWS irrelevant by replacing current version newsfile.write(GOOD_NEWS.replace('0.0.1', '0.0.0')) - self.assertEqual(commitmessage.process_NEWS('NEWS'), ([], set())) + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), ([], set())) def test_process_NEWS_good_cves(self): """ @@ -90,7 +90,7 @@ class TestCommitmessage(unittest.TestCase): .replace('change2.2', 'CVE-2-2')\ .split('\n')[3:13] expected_cvs = set(['CVE-2-1', 'CVE-2-2']) - self.assertEqual(commitmessage.process_NEWS('NEWS'), + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), (expected_msg, expected_cvs)) def test_process_NEWS_long(self): @@ -111,7 +111,7 @@ class TestCommitmessage(unittest.TestCase): expected_msg.extend(['1', '2', '3', '4', '5', '6', '7', '', '(NEWS truncated at 15 lines)', '']) expected_cvs = set() - self.assertEqual(commitmessage.process_NEWS('NEWS'), + self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), (expected_msg, expected_cvs)) def test_guess_commit_message(self): @@ -119,9 +119,11 @@ class TestCommitmessage(unittest.TestCase): Test guess_commit_message() with mocked internal functions and both commitmessage information and cves available from newsfile. """ + conf = config.Config() + conf.old_version = "0.0.0" process_NEWS_backup = commitmessage.process_NEWS - def mock_process_NEWS(newsfile): + def mock_process_NEWS(newsfile, old_version): return (['', 'commit', 'message', 'with', 'cves', ''], set(['cve1', 'cve2'])) @@ -129,7 +131,7 @@ class TestCommitmessage(unittest.TestCase): open_name = 'util.open_auto' with mock.patch(open_name, create=True) as mock_open: mock_open.return_value = mock.MagicMock() - commitmessage.guess_commit_message("") + commitmessage.guess_commit_message("", conf) # reset mocks before asserting so a failure doesn't cascade to # other tests commitmessage.process_NEWS = process_NEWS_backup @@ -147,23 +149,23 @@ class TestCommitmessage(unittest.TestCase): also available from config, which changes the first line of the commmit message. """ + conf = config.Config() process_NEWS_backup = commitmessage.process_NEWS - def mock_process_NEWS(newsfile): + def mock_process_NEWS(newsfile, old_version): return (['', 'commit', 'message', 'with', 'cves', ''], set(['cve1', 'cve2'])) commitmessage.process_NEWS = mock_process_NEWS - commitmessage.config.cves = set(['CVE-1234-5678']) - commitmessage.config.old_version = None # Allow cve title to be set + conf.cves = set(['CVE-1234-5678']) + conf.old_version = None # Allow cve title to be set open_name = 'util.open_auto' with mock.patch(open_name, create=True) as mock_open: mock_open.return_value = mock.MagicMock() - commitmessage.guess_commit_message("") + commitmessage.guess_commit_message("", conf) # reset mocks before asserting so a failure doesn't cascade to # other tests commitmessage.process_NEWS = process_NEWS_backup - commitmessage.config.cves = set() fh = mock_open.return_value.__enter__.return_value fh.write.assert_called_with( 'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n' @@ -178,23 +180,23 @@ class TestCommitmessage(unittest.TestCase): message. Additionally there is imported key info that will be displayed at the end of the message. """ + conf = config.Config() process_NEWS_backup = commitmessage.process_NEWS - def mock_process_NEWS(newsfile): + def mock_process_NEWS(newsfile, old_version): return (['', 'commit', 'message', 'with', 'cves', ''], set(['cve1', 'cve2'])) commitmessage.process_NEWS = mock_process_NEWS - commitmessage.config.cves = set(['CVE-1234-5678']) - commitmessage.config.old_version = None # Allow cve title to be set + conf.cves = set(['CVE-1234-5678']) + conf.old_version = None # Allow cve title to be set open_name = 'util.open_auto' with mock.patch(open_name, create=True) as mock_open: mock_open.return_value = mock.MagicMock() - commitmessage.guess_commit_message("keyinfo content") + commitmessage.guess_commit_message("keyinfo content", conf) # reset mocks before asserting so a failure doesn't cascade to # other tests commitmessage.process_NEWS = process_NEWS_backup - commitmessage.config.cves = set() fh = mock_open.return_value.__enter__.return_value fh.write.assert_called_with( 'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n' @@ -206,12 +208,13 @@ class TestCommitmessage(unittest.TestCase): """ Tests scan_for_changes using temporary directories """ + conf = config.Config() with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'changelog.txt'), 'w') as newsfile: newsfile.write('new changelog file') with tempfile.TemporaryDirectory() as tmpd1: - commitmessage.scan_for_changes(tmpd1, tmpd) + commitmessage.scan_for_changes(tmpd1, tmpd, conf.transforms) self.assertTrue(os.path.isfile(tmpd1 + '/ChangeLog')) diff --git a/tests/test_files.py b/tests/test_files.py index fa7b011..75bb115 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -20,7 +20,8 @@ def mock_return(retval): class TestFiles(unittest.TestCase): def setUp(self): - self.fm = FileManager() + conf = config.Config() + self.fm = FileManager(conf) def test_banned_path(self): """ @@ -66,14 +67,14 @@ class TestFiles(unittest.TestCase): """ Test compat_exclude with a file that shouldn't be excluded. """ - config.config_opts['compat'] = True + self.fm.config.config_opts['compat'] = True self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so.1')) def test_compat_exclude_exclude_file(self): """ Test compat_exclude with a file that should be excluded. """ - config.config_opts['compat'] = True + self.fm.config.config_opts['compat'] = True self.assertTrue(self.fm.compat_exclude('/usr/lib64/libfoo.so')) def test_compat_exclude_not_compat_mode(self): @@ -81,7 +82,7 @@ class TestFiles(unittest.TestCase): Test compat_exclude with a file that should be excluded but isn't because the package isn't being run in compat mode. """ - config.config_opts['compat'] = False + self.fm.config.config_opts['compat'] = False self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so')) def test_file_pat_match(self): diff --git a/tests/test_infile_update_spec.py b/tests/test_infile_update_spec.py index 72cf27d..b10d2a2 100644 --- a/tests/test_infile_update_spec.py +++ b/tests/test_infile_update_spec.py @@ -1,5 +1,6 @@ import unittest +import config import infile_update_spec import specfiles @@ -8,7 +9,7 @@ class TestUpdateSpecfile(unittest.TestCase): def setUp(self): # url, version, name, release url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1') + self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config()) self.bb_dict = { "DEPENDS": "ncurses gettext-native", diff --git a/tests/test_license.py b/tests/test_license.py index b7ba083..9fcf424 100644 --- a/tests/test_license.py +++ b/tests/test_license.py @@ -7,6 +7,7 @@ from unittest.mock import patch, mock_open, MagicMock import pycurl +import config import download import license import util @@ -14,20 +15,17 @@ import util class TestLicense(unittest.TestCase): - @classmethod - def setUpClass(self): - license.config.setup_patterns() - def setUp(self): license.licenses = [] - license.config.license_fetch = None def test_add_license(self): """ Test add_license from valid string, Apache-2 should be translated to Apache-2.0 """ - self.assertTrue(license.add_license('Apache-2')) + conf = config.Config() + conf.setup_patterns() + self.assertTrue(license.add_license('Apache-2', conf.license_translations, conf.license_blacklist)) self.assertIn('Apache-2.0', license.licenses) def test_add_license_present(self): @@ -36,8 +34,10 @@ class TestLicense(unittest.TestCase): the licenses list. Should return True and should not modify the licenses list. GPL-3 translates to GPL-3.0. """ + conf = config.Config() + conf.setup_patterns() license.licenses.append('GPL-3.0') - self.assertTrue(license.add_license('GPL-3')) + self.assertTrue(license.add_license('GPL-3', conf.license_translations, conf.license_blacklist)) self.assertEqual(['GPL-3.0'], license.licenses) def test_add_license_blacklisted(self): @@ -45,18 +45,22 @@ class TestLicense(unittest.TestCase): Test add_license from string in license_blacklist. Should return False and should not modify the licenses list. """ + conf = config.Config() + conf.setup_patterns() # sanity check to make sure the licenses list is empty before the later # assertIn() call self.assertEqual(license.licenses, []) - self.assertFalse(license.add_license('License')) + self.assertFalse(license.add_license('License', conf.license_translations, conf.license_blacklist)) self.assertNotIn('License', license.licenses) def test_license_from_copying_hash(self): """ Test license_from_copying_hash with valid license file """ - license.license_from_copying_hash('tests/COPYING_TEST', '') + conf = config.Config() + conf.setup_patterns() + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) self.assertIn('GPL-3.0', license.licenses) def test_license_from_copying_hash_no_license_show(self): @@ -64,26 +68,26 @@ class TestLicense(unittest.TestCase): Test license_from_copying_hash with invalid hash and no license_show set """ - bkup_hash = license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')] + conf = config.Config() + conf.setup_patterns() # remove the hash from license_hashes - del(license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')]) - license.config.license_show = "license.show.url" - license.license_from_copying_hash('tests/COPYING_TEST', '') + del(conf.license_hashes[license.get_sha1sum('tests/COPYING_TEST')]) + conf.license_show = "license.show.url" + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) - # restore the hash - license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')] = bkup_hash self.assertEquals(license.licenses, []) def test_license_from_copying_hash_bad_license(self): """ Test license_from_copying_hash with invalid license file """ + conf = config.Config() content = util.get_contents("tests/COPYING_TEST").replace(b"GNU", b"SNU") m_open = MagicMock() m_open.__str__.return_value = content with patch('license.get_contents', m_open, create=True): - license.license_from_copying_hash('copying.txt', '') + license.license_from_copying_hash('copying.txt', '', conf) self.assertEquals(license.licenses, []) @@ -114,13 +118,14 @@ class TestLicense(unittest.TestCase): # set the mock curl download.pycurl.Curl = MockCurl - license.config.license_fetch = 'license.server.url' + conf = config.Config() + conf.license_fetch = 'license.server.url' # let's check that the proper thing is being printed as well out = StringIO() with redirect_stdout(out): with self.assertRaises(SystemExit): - license.license_from_copying_hash('tests/COPYING_TEST', '') + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) self.assertIn('Unable to fetch license.server.url: Test Exception', out.getvalue()) @@ -167,12 +172,13 @@ class TestLicense(unittest.TestCase): # set the mock curl download.pycurl.Curl = MockCurl - license.config.license_fetch = 'license.server.url' + conf = config.Config() + conf.license_fetch = 'license.server.url' # let's check that the proper thing is being printed as well out = StringIO() with redirect_stdout(out): - license.license_from_copying_hash('tests/COPYING_TEST', '') + license.license_from_copying_hash('tests/COPYING_TEST', '', conf) self.assertIn('GPL-3.0', license.licenses) self.assertIn('License : GPL-3.0 (server)', out.getvalue()) @@ -187,6 +193,8 @@ class TestLicense(unittest.TestCase): """ Test scan_for_licenses in temporary directory with valid license file """ + conf = config.Config() + conf.setup_patterns() with open('tests/COPYING_TEST', 'rb') as copyingf: content = copyingf.read() @@ -198,7 +206,7 @@ class TestLicense(unittest.TestCase): for testf in ['testlib.c', 'testmain.c', 'testheader.h']: with open(os.path.join(tmpd, testf), 'w') as newtestf: newtestf.write('test content') - license.scan_for_licenses(tmpd) + license.scan_for_licenses(tmpd, conf) self.assertIn('GPL-3.0', license.licenses) @@ -208,6 +216,8 @@ class TestLicense(unittest.TestCase): Should not add any licenses, should print a fatal message, should exit with a status code of 1. """ + conf = config.Config() + conf.setup_patterns() with tempfile.TemporaryDirectory() as tmpd: # create some cruft for testing for testf in ['testlib.c', 'testmain.c', 'testheader.h']: @@ -217,7 +227,7 @@ class TestLicense(unittest.TestCase): out = StringIO() with redirect_stdout(out): with self.assertRaises(SystemExit) as thread: - license.scan_for_licenses(tmpd) + license.scan_for_licenses(tmpd, conf) self.assertEqual(thread.exception.code, 1) self.assertIn("Cannot find any license", out.getvalue()) diff --git a/tests/test_pkg_integrity.py b/tests/test_pkg_integrity.py index cc4a1c2..f42dd53 100644 --- a/tests/test_pkg_integrity.py +++ b/tests/test_pkg_integrity.py @@ -6,6 +6,7 @@ import tempfile import unittest from unittest.mock import patch +import config import download import pkg_integrity @@ -46,27 +47,30 @@ def mock_download_do_curl(url, dst=None): @patch('download.do_curl', mock_download_do_curl) class TestCheckFn(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False + def _mock_rewrite(self, path): + del path def test_check_matching_sign_url(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: pkey = "023A4420C7EC6914.pkey" shutil.copy(os.path.join(TESTKEYDIR, pkey), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd) - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertTrue(result) def test_check_with_existing_sign(self): """ Download signature for local verification """ + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd) - result = pkg_integrity.check(NOSIGN_PKT_URL, tmpd) + result = pkg_integrity.check(NOSIGN_PKT_URL, tmpd, conf) self.assertTrue(result) @@ -130,11 +134,8 @@ class TestDomainBasedVerifiers(unittest.TestCase): @patch('download.do_curl', mock_download_do_curl) class TestGEMShaVerifier(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False + def _mock_rewrite(self, path): + del path def _mock_get_gem_info(pkg): info = ''' @@ -153,67 +154,85 @@ class TestGEMShaVerifier(unittest.TestCase): @patch('pkg_integrity.GEMShaVerifier.get_rubygems_info', _mock_get_gem_info) def test_from_url(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: filen = os.path.basename(GEM_PKT) shutil.copy(os.path.join(TESTDIR, filen), tmpd) - result = pkg_integrity.check(GEM_PKT, tmpd) + result = pkg_integrity.check(GEM_PKT, tmpd, conf) self.assertTrue(result) @patch('pkg_integrity.GEMShaVerifier.get_rubygems_info', _mock_get_gem_info) def test_non_matchingsha(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: out_file = os.path.join(tmpd, os.path.basename(GEM_PKT)) f = open(out_file, 'wb') f.write(b'this is made up data that will force a failure') f.close() with self.assertRaises(SystemExit) as a: - pkg_integrity.check(GEM_PKT, tmpd) + pkg_integrity.check(GEM_PKT, tmpd, conf) self.assertEqual(a.exception.code, 1) @patch('download.do_curl', mock_download_do_curl) class TestGPGVerifier(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False + def _mock_rewrite(self, path): + del path def test_from_url(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertTrue(result) def test_invalid_key(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd) with open(os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD) + ".asc"), 'w') as ofile: ofile.write("Invalid signature") - result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd) + result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd, conf) self.assertIsNone(result) def test_key_not_found(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd) - result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd) + result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd, conf) self.assertIsNone(result) def test_from_disk(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL)) out_key = out_file + ".asc" shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd) - result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key) + result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key, conf) self.assertTrue(result) def test_non_matchingsig(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL)) @@ -221,19 +240,26 @@ class TestGPGVerifier(unittest.TestCase): f.write(b'made up date that will fail check') f.close() with self.assertRaises(SystemExit) as a: - pkg_integrity.check(PACKAGE_URL, tmpd) + pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertEqual(a.exception.code, 1) def test_result_on_non_existent_pkg_path(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz', 'NonExistentPKG.tar.gz', - 'NonExistentKey.asc') + 'NonExistentKey.asc', + conf) self.assertIsNone(result) def test_result_on_nosign_package(self): + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd) - result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd) + result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd, conf) self.assertIsNone(result) @patch.object(pkg_integrity.GPGCli, 'exec_cmd') @@ -270,10 +296,13 @@ class TestGPGVerifier(unittest.TestCase): mock_exec.return_value = (b'', b'', 0) + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertTrue(result) self.assertEqual(mock_parse.call_count, 4) self.assertEqual(mock_exec.call_count, 3) @@ -308,11 +337,14 @@ class TestGPGVerifier(unittest.TestCase): mock_exec.return_value = (b'', b'', 0) + conf = config.Config() + conf.rewrite_config_opts = self._mock_rewrite + conf.config_opts['verify_required'] = False with tempfile.TemporaryDirectory() as tmpd: shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd) shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd) with self.assertRaises(SystemExit) as msg: - result = pkg_integrity.check(PACKAGE_URL, tmpd) + result = pkg_integrity.check(PACKAGE_URL, tmpd, conf) self.assertEqual(msg.exception.code, 1) self.assertEqual(mock_parse.call_count, 4) self.assertEqual(mock_exec.call_count, 2) @@ -335,12 +367,6 @@ class TestInputGetter(unittest.TestCase): class TestUtils(unittest.TestCase): - def setUp(self): - def mock_rewrite(path): - del path - pkg_integrity.config.rewrite_config_opts = mock_rewrite - pkg_integrity.config.config_opts['verify_required'] = False - def test_get_verifier(self): x = pkg_integrity.get_verifier('file.abcd') self.assertEqual(x, None) diff --git a/tests/test_specdescription.py b/tests/test_specdescription.py index 76fab91..80d882b 100644 --- a/tests/test_specdescription.py +++ b/tests/test_specdescription.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import mock_open, patch +import config import specdescription @@ -15,7 +16,6 @@ class TestSpecdescription(unittest.TestCase): specdescription.default_summary = "No detailed summary available" specdescription.default_summary_score = 0 specdescription.license.licenses = [] - specdescription.config.license_translations = {} def test_clean_license_string(self): """ @@ -48,7 +48,7 @@ class TestSpecdescription(unittest.TestCase): "%donewithdesc\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_spec("filename") + specdescription.description_from_spec("filename", {}, []) self.assertEqual(specdescription.default_description, "Here is the description section\n" @@ -68,7 +68,7 @@ class TestSpecdescription(unittest.TestCase): content = "# this is a specfile without much in it\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_spec("filename") + specdescription.description_from_spec("filename", {}, []) self.assertEqual(specdescription.default_description, "No detailed description available") @@ -94,7 +94,7 @@ class TestSpecdescription(unittest.TestCase): "donewithdesc:\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_pkginfo("filename") + specdescription.description_from_pkginfo("filename", {}, []) self.assertEqual(specdescription.default_description, "Here is the description section\n" @@ -113,7 +113,7 @@ class TestSpecdescription(unittest.TestCase): content = "# this is a pkginfo file without much in it\n" m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): - specdescription.description_from_pkginfo("filename") + specdescription.description_from_pkginfo("filename", {}, []) self.assertEqual(specdescription.default_description, "No detailed description available") diff --git a/tests/test_specfile.py b/tests/test_specfile.py index 2df2268..353d836 100644 --- a/tests/test_specfile.py +++ b/tests/test_specfile.py @@ -1,5 +1,6 @@ import unittest import unittest.mock +import config import specfiles @@ -11,9 +12,10 @@ class TestSpecfileWrite(unittest.TestCase): self.WRITES = [] def setUp(self): + conf = config.Config() + conf.config_opts['dev_requires_extras'] = False url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2') - specfiles.config.config_opts['dev_requires_extras'] = False + self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf) def mock_write(string): self.WRITES.append(string) @@ -286,8 +288,6 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_scriplets base test """ - backup_read_conf_file = specfiles.config.read_conf_file - def mock_read_conf_file(name): prefix = "pre" if "pre" in name else "post" return ["{}-script line 1\n".format(prefix), @@ -299,12 +299,11 @@ class TestSpecfileWrite(unittest.TestCase): def writelines(stringlist): self.WRITES.extend(stringlist) - specfiles.config.read_conf_file = mock_read_conf_file + self.specfile.config.read_conf_file = mock_read_conf_file self.specfile.specfile = MockSpecfile() self.specfile.packages["ruby"] = ["rubyfile1"] self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"] self.specfile.write_scriplets() - specfiles.config.read_conf_file = backup_read_conf_file expect = ["\n%post ruby\n", "post-script line 1\n\n", "post-script line 2\n\n", @@ -320,8 +319,6 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_scriplets with only pre-scripts present. """ - backup_read_conf_file = specfiles.config.read_conf_file - def mock_read_conf_file(name): if "pre" in name: return ["pre-script line 1\n", @@ -335,12 +332,11 @@ class TestSpecfileWrite(unittest.TestCase): def writelines(stringlist): self.WRITES.extend(stringlist) - specfiles.config.read_conf_file = mock_read_conf_file + self.specfile.config.read_conf_file = mock_read_conf_file self.specfile.specfile = MockSpecfile() self.specfile.packages["ruby"] = ["rubyfile1"] self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"] self.specfile.write_scriplets() - specfiles.config.read_conf_file = backup_read_conf_file expect = ["\n%pre ruby\n", "pre-script line 1\n\n", "pre-script line 2\n\n", @@ -352,8 +348,6 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_scriplets with no scripts present. """ - backup_read_conf_file = specfiles.config.read_conf_file - def mock_read_conf_file(name): return [] @@ -362,12 +356,11 @@ class TestSpecfileWrite(unittest.TestCase): def writelines(stringlist): self.WRITES.extend(stringlist) - specfiles.config.read_conf_file = mock_read_conf_file + self.specfile.config.read_conf_file = mock_read_conf_file self.specfile.specfile = MockSpecfile() self.specfile.packages["ruby"] = ["rubyfile1"] self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"] self.specfile.write_scriplets() - specfiles.config.read_conf_file = backup_read_conf_file self.assertEqual([], self.WRITES) def test_write_files_base(self): diff --git a/tests/test_tarball.py b/tests/test_tarball.py index 6ed8ca3..4f31151 100644 --- a/tests/test_tarball.py +++ b/tests/test_tarball.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import Mock, patch +import config import tarball @@ -123,11 +124,12 @@ def detect_build_test_generator(url, build_pattern): def name_and_version_test_generator(url, name, version): """Create test for tarball.name_and_version method.""" - @patch('tarball.config.parse_config_versions', Mock(return_value={})) def generator(self): """Test template.""" + conf = config.Config() + conf.parse_config_versions = Mock(return_value={}) tarball.url = url - n, _, v = tarball.name_and_version('', '', Mock()) + n, _, v = tarball.name_and_version('', '', Mock(), conf) self.assertEqual(name, n) self.assertEqual(version, v) if "github.com" in url: @@ -217,7 +219,6 @@ class TestTarball(unittest.TestCase): tarball.process_go_archives(go_archives) self.assertEqual(go_archives, go_archives_expected) - @patch('tarball.config', Mock()) def test_process_multiver_archives(self): """Test for tarball.process_multiver_archives method.""" # Set up input values @@ -236,8 +237,9 @@ class TestTarball(unittest.TestCase): ] # Set up a return value for parse_config_versions method attrs = {'parse_config_versions.return_value': config_versions} - tarball.config.configure_mock(**attrs) - tarball.process_multiver_archives(main_src, multiver_archives) + conf = Mock() + conf.configure_mock(**attrs) + tarball.process_multiver_archives(main_src, multiver_archives, conf) self.assertEqual(multiver_archives, expected_multiver_archives) @patch('tarball.Source.set_prefix', Mock())