clearlinux/autospec
2
0
Fork 0
mirror of https://github.com/clearlinux/autospec.git synced 2026-06-16 02:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor the config into a class

Browse Source 
The config module had a large amount of globals that were being
touched across many modules that would import. This made changes to
config very fragile as figuring out what would be modified in any
given call chain was difficult to diagnose.

It also made testing fragile as one would need to reset a given
module's config import to the best of their knowledge before rerunning
another test.

To get away from that (and to try and reduce the number of globally
modified variablies in autospec as a whole), refactor the config
module to provide its state as part config class. The long running
goal of changes like this is to better track what content can be
updated by a particular function (if a function would have access
to the config instance is now the hint rather than the config module
getting imported).
This commit is contained in:
William Douglas
2020-03-19 16:45:13 -07:00
committed by Patrick McCarty
parent 0e5f88efe0
commit 54636b48dd
25 changed files with 1370 additions and 1402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
autospec/autospec.py
+32 -31
View File
@@ -85,11 +85,11 @@ def check_requirements(use_git):
sys.exit(1)
def load_specfile(specfile):
def load_specfile(conf, specfile):
"""Gather all information from static analysis into Specfile instance."""
config.load_specfile(specfile)
conf.load_specfile(specfile)
tarball.load_specfile(specfile)
specdescription.load_specfile(specfile)
specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ)
license.load_specfile(specfile)
buildreq.load_specfile(specfile)
buildpattern.load_specfile(specfile)
@@ -125,10 +125,10 @@ def save_mock_logs(path, iteration):
os.rename(src, dest)
def write_prep(workingdir):
def write_prep(conf, workingdir):
"""Write metadata to the local workingdir when --prep-only is used."""
if config.urlban:
used_url = re.sub(config.urlban, "localhost", tarball.url)
if conf.urlban:
used_url = re.sub(conf.urlban, "localhost", tarball.url)
else:
used_url = tarball.url
@@ -239,6 +239,7 @@ def main():
def package(args, url, name, archives, workingdir, infile_dict):
"""Entry point for building a package with autospec."""
conf = config.Config()
check_requirements(args.git)
build.setup_workingdir(workingdir)
@@ -246,9 +247,9 @@ def package(args, url, name, archives, workingdir, infile_dict):
# First, download the tarball, extract it and then do a set
# of static analysis on the content of the tarball.
#
filemanager = files.FileManager()
tarball.process(url, name, args.version, args.target, archives, filemanager)
config.create_versions(build.download_path, tarball.multi_version)
filemanager = files.FileManager(conf)
tarball.process(url, name, args.version, args.target, archives, filemanager, conf)
conf.create_versions(build.download_path, tarball.multi_version)
# Search up one level from here to capture multiple versions
_dir = tarball.path
@@ -262,36 +263,36 @@ def package(args, url, name, archives, workingdir, infile_dict):
except Exception:
pass
# Start one directory higher so we scan *all* versions for licenses
license.scan_for_licenses(os.path.dirname(_dir))
license.scan_for_licenses(os.path.dirname(_dir), conf)
exit(0)
config.setup_patterns()
config.config_file = args.config
config.parse_config_files(build.download_path, args.bump, filemanager, tarball.version)
config.setup_patterns(config.failed_pattern_dir)
config.parse_existing_spec(build.download_path, tarball.name)
conf.setup_patterns()
conf.config_file = args.config
conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version)
conf.setup_patterns(conf.failed_pattern_dir)
conf.parse_existing_spec(build.download_path, tarball.name)
if args.prep_only:
write_prep(workingdir)
write_prep(conf, workingdir)
exit(0)
buildreq.set_build_req()
buildreq.scan_for_configure(_dir, tarball.name, build.download_path)
specdescription.scan_for_description(tarball.name, _dir)
buildreq.scan_for_configure(_dir, tarball.name, build.download_path, conf)
specdescription.scan_for_description(tarball.name, _dir, conf.license_translations, conf.license_blacklist)
# Start one directory higher so we scan *all* versions for licenses
license.scan_for_licenses(os.path.dirname(_dir))
commitmessage.scan_for_changes(build.download_path, _dir)
license.scan_for_licenses(os.path.dirname(_dir), conf)
commitmessage.scan_for_changes(build.download_path, _dir, conf.transforms)
add_sources(build.download_path, archives)
check.scan_for_tests(_dir)
check.scan_for_tests(_dir, conf)
#
# Now, we have enough to write out a specfile, and try to build it.
# We will then analyze the build result and learn information until the
# package builds
#
specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release)
specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release, conf)
filemanager.load_specfile(specfile)
load_specfile(specfile)
load_specfile(conf, specfile)
#
# If infile is passed, parse it and overwrite the specfile configurations
@@ -303,12 +304,12 @@ def package(args, url, name, archives, workingdir, infile_dict):
if args.integrity:
interactive_mode = not args.non_interactive
pkg_integrity.check(url, build.download_path, interactive=interactive_mode)
pkg_integrity.check(url, build.download_path, conf, interactive=interactive_mode)
pkg_integrity.load_specfile(specfile)
specfile.write_spec(build.download_path)
while 1:
build.package(filemanager, args.mock_config, args.mock_opts, args.cleanup)
build.package(filemanager, args.mock_config, args.mock_opts, conf, args.cleanup)
filemanager.load_specfile(specfile)
specfile.write_spec(build.download_path)
filemanager.newfiles_printed = 0
@@ -326,10 +327,10 @@ def package(args, url, name, archives, workingdir, infile_dict):
save_mock_logs(build.download_path, build.round)
check.check_regression(build.download_path)
check.check_regression(build.download_path, conf.config_opts['skip_tests'])
if build.success == 0:
config.create_buildreq_cache(build.download_path, tarball.version)
conf.create_buildreq_cache(build.download_path, tarball.version)
print_fatal("Build failed, aborting")
sys.exit(1)
elif os.path.isfile("README.clear"):
@@ -345,18 +346,18 @@ def package(args, url, name, archives, workingdir, infile_dict):
examine_abi(build.download_path)
if os.path.exists("/var/lib/rpm"):
pkg_scan.get_whatrequires(tarball.name)
pkg_scan.get_whatrequires(tarball.name, conf.yum_conf)
write_out(build.download_path + "/release", tarball.release + "\n")
# record logcheck output
logcheck(build.download_path)
commitmessage.guess_commit_message(pkg_integrity.IMPORTED)
config.create_buildreq_cache(build.download_path, tarball.version)
commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf)
conf.create_buildreq_cache(build.download_path, tarball.version)
if args.git:
git.commit_to_git(build.download_path)
git.commit_to_git(build.download_path, conf)
else:
print("To commit your changes, git add the relevant files and "
"run 'git commit -F commitmsg'")
autospec/build.py
+11 -12
View File
@@ -24,7 +24,6 @@ import re
import shutil
import buildreq
import config
import tarball
import util
@@ -45,13 +44,13 @@ def setup_workingdir(workingdir):
download_path = os.path.join(base_path, tarball.name)
def simple_pattern_pkgconfig(line, pattern, pkgconfig):
def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32):
"""Check for pkgconfig patterns and restart build as needed."""
global must_restart
pat = re.compile(pattern)
match = pat.search(line)
if match:
must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, cache=True)
must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True)
def simple_pattern(line, pattern, req):
@@ -97,7 +96,7 @@ def cleanup_req(s: str) -> str:
return s
def failed_pattern(line, pattern, verbose, buildtool=None):
def failed_pattern(line, config, pattern, verbose, buildtool=None):
"""Check against failed patterns to restart build as needed."""
global must_restart
global warned_about
@@ -119,11 +118,11 @@ def failed_pattern(line, pattern, verbose, buildtool=None):
if req:
must_restart += buildreq.add_buildreq(req, cache=True)
elif buildtool == 'pkgconfig':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True)
elif buildtool == 'R':
if buildreq.add_buildreq("R-" + s, cache=True) > 0:
must_restart += 1
buildreq.add_requires("R-" + s)
buildreq.add_requires("R-" + s, config.os_packages)
elif buildtool == 'perl':
s = s.replace('inc::', '')
must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True)
@@ -167,7 +166,7 @@ def failed_pattern(line, pattern, verbose, buildtool=None):
# Fallback to mvn-ARTIFACTID package name
must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True)
elif buildtool == 'catkin':
must_restart += buildreq.add_pkgconfig_buildreq(s, cache=True)
must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True)
must_restart += buildreq.add_buildreq(s, cache=True)
except Exception:
@@ -207,7 +206,7 @@ def check_for_warning_pattern(line):
util.print_warning("Build log contains: {}".format(pat))
def parse_build_results(filename, returncode, filemanager):
def parse_build_results(filename, returncode, filemanager, config):
"""Handle build log contents."""
global must_restart
global success
@@ -222,13 +221,13 @@ def parse_build_results(filename, returncode, filemanager):
for line in loglines:
for pat in config.pkgconfig_pats:
simple_pattern_pkgconfig(line, *pat)
simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit'))
for pat in config.simple_pats:
simple_pattern(line, *pat)
for pat in config.failed_pats:
failed_pattern(line, *pat)
failed_pattern(line, config, *pat)
check_for_warning_pattern(line)
@@ -277,7 +276,7 @@ def get_mock_cmd():
return 'sudo /usr/bin/mock'
def package(filemanager, mockconfig, mockopts, cleanup=False):
def package(filemanager, mockconfig, mockopts, config, cleanup=False):
"""Run main package build routine."""
global round
global uniqueext
@@ -343,7 +342,7 @@ def package(filemanager, mockconfig, mockopts, cleanup=False):
is_clean = parse_buildroot_log(download_path + "/results/root.log", ret)
if is_clean:
parse_build_results(download_path + "/results/build.log", ret, filemanager)
parse_build_results(download_path + "/results/build.log", ret, filemanager, config)
if filemanager.has_banned:
util.print_fatal("Content in banned paths found, aborting build")
exit(1)
autospec/buildreq.py
+47 -49
View File
@@ -23,10 +23,8 @@ import ast
import json
import os
import re
import subprocess
import buildpattern
import config
import pypidata
import specdescription
import toml
@@ -83,8 +81,8 @@ def add_buildreq(req, cache=False):
return new
def add_requires(req, override=False):
"""Add req to the global requires set if it is present in buildreqs and os_packages and is not banned."""
def add_requires(req, packages, override=False):
"""Add req to the global requires set if it is present in buildreqs and packages and is not banned."""
global buildreqs
global requires
new = True
@@ -96,7 +94,7 @@ def add_requires(req, override=False):
# Try dashes instead of underscores as some ecosystems are inconsistent in their naming
req2 = req.replace("_", "-")
if req not in buildreqs and req2 in config.os_packages and req2 not in requires and req2 not in banned_requires:
if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires:
# Since this is done for python add a buildreq just in case (might not be correct though)
buildreqs.add(req2)
requires.add(req2)
@@ -108,13 +106,13 @@ def add_requires(req, override=False):
req2 = req[0].lower() + req[1:]
else:
req2 = req[0].upper() + req[1:]
if req not in buildreqs and req2 in config.os_packages and req2 not in requires and req2 not in banned_requires:
if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires:
# Since this is done for python add a buildreq just in case (might not be correct though)
buildreqs.add(req2)
requires.add(req2)
return True
if req not in buildreqs and req not in config.os_packages and not override:
if req not in buildreqs and req not in packages and not override:
if req:
print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req))
return False
@@ -124,16 +122,16 @@ def add_requires(req, override=False):
return new
def add_pkgconfig_buildreq(preq, cache=False):
def add_pkgconfig_buildreq(preq, conf32, cache=False):
"""Format preq as pkgconfig req and add to buildreqs."""
if config.config_opts['32bit']:
if conf32:
req = "pkgconfig(32" + preq + ")"
add_buildreq(req, cache)
req = "pkgconfig(" + preq + ")"
return add_buildreq(req, cache)
def configure_ac_line(line):
def configure_ac_line(line, conf32):
"""Parse configure_ac line and add appropriate buildreqs."""
# print("----\n", line, "\n----")
# ignore comments
@@ -164,7 +162,7 @@ def configure_ac_line(line):
if len(L) > 1:
rqlist = L[1].strip()
for req in parse_modules_list(rqlist):
add_pkgconfig_buildreq(req)
add_pkgconfig_buildreq(req, conf32)
# PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found)
match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line)
@@ -172,7 +170,7 @@ def configure_ac_line(line):
L = match.group(1).split(",")
rqlist = L[0].strip()
for req in parse_modules_list(rqlist):
add_pkgconfig_buildreq(req)
add_pkgconfig_buildreq(req, conf32)
def is_number(num_str):
@@ -269,7 +267,7 @@ def parse_modules_list(modules_string, is_cmake=False):
return res
def parse_configure_ac(filename):
def parse_configure_ac(filename, conf32):
"""Parse the configure.ac file for build requirements."""
buf = ""
depth = 0
@@ -287,14 +285,14 @@ def parse_configure_ac(filename):
if c != "\n":
buf += c
if c == "\n" and depth == 0:
configure_ac_line(buf)
configure_ac_line(buf, conf32)
buf = ""
configure_ac_line(buf)
configure_ac_line(buf, conf32)
f.close()
def parse_cargo_toml(filename):
def parse_cargo_toml(filename, packages):
"""Update build requirements using Cargo.toml.
Set the build requirements for building rust programs using cargo.
@@ -310,7 +308,7 @@ def parse_cargo_toml(filename):
return
for cdep in cargo["dependencies"]:
if add_buildreq(cdep):
add_requires(cdep)
add_requires(cdep, packages)
def _get_desc_field(field, desc):
@@ -402,7 +400,7 @@ def _get_r_provides():
return set(provides)
def parse_r_description(filename):
def parse_r_description(filename, packages):
"""Update build/runtime requirements according to the R package description."""
deps = []
with util.open_auto(filename, "r") as desc:
@@ -417,9 +415,9 @@ def parse_r_description(filename):
if dep in r_provides:
continue
pkg = 'R-' + dep
if pkg in config.os_packages:
if pkg in packages:
add_buildreq(pkg)
add_requires(pkg)
add_requires(pkg, packages)
else:
print("CRAN package '{}' not found in os_packages, skipping".format(pkg))
@@ -463,7 +461,7 @@ def set_build_req():
add_buildreq("rustc")
def rakefile(filename):
def rakefile(filename, gems):
"""Scan Rakefile for build requirements."""
with util.open_auto(filename, "r") as f:
lines = f.readlines()
@@ -473,14 +471,14 @@ def rakefile(filename):
match = pat.search(line)
if match:
s = match.group(1)
if s != "rubygems" and s in config.gems:
print("Rakefile-dep: " + config.gems[s])
add_buildreq(config.gems[s])
if s != "rubygems" and s in gems:
print("Rakefile-dep: " + gems[s])
add_buildreq(gems[s])
else:
print("Rakefile-new: rubygem-" + s)
def parse_cmake(filename):
def parse_cmake(filename, cmake_modules, conf32):
"""Scan a .cmake or CMakeLists.txt file for what's it's actually looking for."""
findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I)
pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I)
@@ -495,7 +493,7 @@ def parse_cmake(filename):
if match:
module = match.group(1)
try:
pkg = config.cmake_modules[module]
pkg = cmake_modules[module]
add_buildreq(pkg)
except Exception:
pass
@@ -516,10 +514,10 @@ def parse_cmake(filename):
module = wordmatch.group(2)
# We have a match, so strip out any version info
for m in parse_modules_list(module, is_cmake=True):
add_pkgconfig_buildreq(m)
add_pkgconfig_buildreq(m, conf32)
def qmake_profile(filename):
def qmake_profile(filename, qt_modules):
"""Scan .pro file for build requirements."""
with util.open_auto(filename, "r") as f:
lines = f.readlines()
@@ -533,7 +531,7 @@ def qmake_profile(filename):
for module in s.split():
module = re.sub('-private$', '', module)
try:
pc = config.qt_modules[module]
pc = qt_modules[module]
add_buildreq('pkgconfig({})'.format(pc))
except Exception:
pass
@@ -575,7 +573,7 @@ def clean_python_req(req, add_python=True):
return ret
def grab_python_requirements(descfile):
def grab_python_requirements(descfile, packages):
"""Add python requirements from requirements.txt file."""
if "/demo/" in descfile:
return
@@ -609,7 +607,7 @@ def grab_python_requirements(descfile):
continue
if clean_python_req(line) == 'mock':
continue
add_requires(clean_python_req(line))
add_requires(clean_python_req(line), packages)
def get_python_build_version_from_classifier(filename):
@@ -627,7 +625,7 @@ def get_python_build_version_from_classifier(filename):
return "distutils3"
def add_setup_py_requires(filename):
def add_setup_py_requires(filename, packages):
"""Detect build requirements listed in setup.py in the install_requires and setup_requires lists.
Handles the following patterns:
@@ -675,7 +673,7 @@ def add_setup_py_requires(filename):
dep = clean_python_req(ast.literal_eval(item), False)
add_buildreq(dep)
if req:
add_requires(dep)
add_requires(dep, packages)
except Exception:
# do not fail, the line contained a variable and
@@ -699,7 +697,7 @@ def add_setup_py_requires(filename):
dep = clean_python_req(ast.literal_eval(line), False)
add_buildreq(dep)
if req:
add_requires(dep)
add_requires(dep, packages)
except Exception:
# Do not fail, just keep looking
@@ -721,7 +719,7 @@ def add_setup_py_requires(filename):
dep = clean_python_req(dep)
add_buildreq(dep)
if req:
add_requires(dep)
add_requires(dep, packages)
except Exception:
# do not fail, the line contained a variable and had to
@@ -729,7 +727,7 @@ def add_setup_py_requires(filename):
pass
def parse_catkin_deps(cmakelists_file):
def parse_catkin_deps(cmakelists_file, conf32):
"""Determine requirements for catkin packages."""
f = util.open_auto(cmakelists_file, "r")
lines = f.readlines()
@@ -746,7 +744,7 @@ def parse_catkin_deps(cmakelists_file):
comp = match.group("comp")
if comp:
for curr in comp.split(" "):
add_pkgconfig_buildreq(curr)
add_pkgconfig_buildreq(curr, conf32)
catkin = True
@@ -767,7 +765,7 @@ def is_qmake_pro(f):
return f.endswith(".pro") and not f.startswith(".")
def scan_for_configure(dirn, tname, dlpath):
def scan_for_configure(dirn, tname, dlpath, config):
"""Scan the package directory for build files to determine build pattern."""
global pypi_provides
global pypi_requires
@@ -789,7 +787,7 @@ def scan_for_configure(dirn, tname, dlpath):
add_buildreq("buildreq-scons")
elif buildpattern.default_pattern == "R":
add_buildreq("buildreq-R")
parse_r_description(os.path.join(dirn, "DESCRIPTION"))
parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages)
elif buildpattern.default_pattern == "phpize":
add_buildreq("buildreq-php")
elif buildpattern.default_pattern == "nginx":
@@ -819,7 +817,7 @@ def scan_for_configure(dirn, tname, dlpath):
pkg = "go-" + req[0].replace("/", "-")
add_buildreq(pkg)
if buildpattern.default_pattern == "godep":
add_requires(pkg)
add_requires(pkg, config.os_packages)
if "CMakeLists.txt" in files and "configure.ac" not in files:
add_buildreq("buildreq-cmake")
@@ -827,7 +825,7 @@ def scan_for_configure(dirn, tname, dlpath):
srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or ".."))
if os.path.samefile(dirpath, srcdir):
parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"))
parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit'))
if "configure" in files and os.access(dirpath + '/configure', os.X_OK):
buildpattern.set_build_pattern("configure", default_score)
@@ -836,11 +834,11 @@ def scan_for_configure(dirn, tname, dlpath):
buildpattern.set_build_pattern("qmake", default_score)
if "requires.txt" in files:
grab_python_requirements(dirpath + '/requires.txt')
grab_python_requirements(dirpath + '/requires.txt', config.os_packages)
if "setup.py" in files:
add_buildreq("buildreq-distutils3")
add_setup_py_requires(dirpath + '/setup.py')
add_setup_py_requires(dirpath + '/setup.py', config.os_packages)
python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py')
buildpattern.set_build_pattern(python_pattern, default_score)
@@ -853,7 +851,7 @@ def scan_for_configure(dirn, tname, dlpath):
buildpattern.set_build_pattern("scons", default_score)
if "requirements.txt" in files:
grab_python_requirements(dirpath + '/requirements.txt')
grab_python_requirements(dirpath + '/requirements.txt', config.os_packages)
if "meson.build" in files:
add_buildreq("buildreq-meson")
@@ -865,13 +863,13 @@ def scan_for_configure(dirn, tname, dlpath):
for name in files:
if name.lower() == "cargo.toml" and dirpath == dirn:
parse_cargo_toml(os.path.join(dirpath, name))
parse_cargo_toml(os.path.join(dirpath, name), config.os_packages)
if name.lower().startswith("configure."):
parse_configure_ac(os.path.join(dirpath, name))
parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit'))
if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby":
rakefile(os.path.join(dirpath, name))
rakefile(os.path.join(dirpath, name), config.gems)
if name.endswith(".pro") and buildpattern.default_pattern == "qmake":
qmake_profile(os.path.join(dirpath, name))
qmake_profile(os.path.join(dirpath, name), config.qt_modules)
if name.lower() == "makefile":
buildpattern.set_build_pattern("make", default_score)
if name.lower() == "autogen.sh":
@@ -880,7 +878,7 @@ def scan_for_configure(dirn, tname, dlpath):
buildpattern.set_build_pattern("cmake", default_score)
if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \
and buildpattern.default_pattern == "cmake":
parse_cmake(os.path.join(dirpath, name))
parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit'))
can_reconf = os.path.exists(os.path.join(dirn, "configure.ac"))
if not can_reconf:
autospec/check.py
+10 -11
View File
@@ -24,7 +24,6 @@ import re
import buildpattern
import buildreq
import config
import count
import tarball
import util
@@ -32,9 +31,9 @@ import util
tests_config = ""
def check_regression(pkg_dir):
def check_regression(pkg_dir, skip_tests):
"""Check the build log for test regressions using the count module."""
if config.config_opts['skip_tests']:
if skip_tests:
return
result = count.parse_log(os.path.join(pkg_dir, "results/build.log"))
@@ -56,11 +55,11 @@ def check_regression(pkg_dir):
util.write_out(os.path.join(pkg_dir, "testresults"), res_str)
def scan_for_tests(src_dir):
def scan_for_tests(src_dir, config):
"""Scan source directory for test files and set tests_config accordingly."""
global tests_config
if config.config_opts['skip_tests'] or tests_config:
if config.config_opts.get('skip_tests') or tests_config:
return
makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
@@ -71,7 +70,7 @@ def scan_for_tests(src_dir):
cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
"make test\nmodule unload openmpi"
if config.config_opts['allow_test_failures']:
if config.config_opts.get('allow_test_failures'):
make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
"make VERBOSE=1 V=1 {}check || :\nmodule unload openmpi".format(makeflags)
cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
@@ -80,7 +79,7 @@ def scan_for_tests(src_dir):
perl_check = "make TEST_VERBOSE=1 test"
setup_check = """PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") python setup.py test"""
meson_check = "meson test -C builddir"
if config.config_opts['allow_test_failures']:
if config.config_opts.get('allow_test_failures'):
make_check += " || :"
cmake_check += " || :"
perl_check += " || :"
@@ -96,17 +95,17 @@ def scan_for_tests(src_dir):
"rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd",
"meson": meson_check,
}
if config.config_opts['32bit']:
if config.config_opts.get('32bit'):
testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
if config.config_opts['use_avx2']:
if config.config_opts.get('use_avx2'):
testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
if config.config_opts['use_avx512']:
if config.config_opts.get('use_avx512'):
testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
if config.config_opts['openmpi']:
if config.config_opts.get('openmpi'):
testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi
autospec/commitmessage.py
+9 -10
View File
@@ -29,25 +29,24 @@ import sys
from subprocess import PIPE, run
import build
import config
import tarball
import util
def scan_for_changes(download_path, directory):
def scan_for_changes(download_path, directory, transforms):
"""Scan for changelogs or news files in the file sources.
Scan for changelogs or news files in the source code and copy them to download_path as their
`config.transform`ed name. The file with the transformed name will later be parsed to find the
`transform`ed name. The file with the transformed name will later be parsed to find the
commit message.
"""
found = []
interests = config.transforms.keys()
interests = transforms.keys()
for dirpath, dirnames, files in os.walk(directory, topdown=False):
hits = [x for x in files if x.lower() in interests and x.lower() not in found]
for item in hits:
source = os.path.join(dirpath, item)
target = os.path.join(download_path, config.transforms[item.lower()])
target = os.path.join(download_path, transforms[item.lower()])
try:
shutil.copy(source, target)
os.chmod(target, 0o644)
@@ -82,7 +81,7 @@ def find_in_line(pattern, line):
return bool(re.search(pattern, line))
def process_NEWS(newsfile):
def process_NEWS(newsfile, old_version):
"""Parse the newfile for relevent changes.
Look for changes and CVE fixes relevant to current version update. This information is returned
@@ -99,7 +98,7 @@ def process_NEWS(newsfile):
success = False
start_found = False
if config.old_version is None or config.old_version == tarball.version:
if old_version is None or old_version == tarball.version:
# no version update, so no information to search for in newsfile
return commitmessage, cves
@@ -113,7 +112,7 @@ def process_NEWS(newsfile):
# escape some values for use in regular expressions below
escaped_curver = re.escape(tarball.version)
escaped_oldver = re.escape(config.old_version)
escaped_oldver = re.escape(old_version)
escaped_tarname = re.escape(tarball.name)
# these are patterns that define the beginning of a block of information
@@ -219,7 +218,7 @@ def process_git(giturl, oldversion, newversion):
return shortlog
def guess_commit_message(keyinfo):
def guess_commit_message(keyinfo, config):
"""Parse newsfile for a commit message.
Try and find a sane commit message for the newsfile. The commit message defaults to the
@@ -271,7 +270,7 @@ def guess_commit_message(keyinfo):
newsfiles = ["NEWS", "ChangeLog"]
for newsfile in newsfiles:
# parse news files for relevant version updates and cve fixes
newcommitmessage, newcves = process_NEWS(newsfile)
newcommitmessage, newcves = process_NEWS(newsfile, config.old_version)
commitmessage.extend(newcommitmessage)
cves.update(newcves)
autospec/config.py
+804 -888
View File
File diff suppressed because it is too large Load Diff
autospec/files.py
+5 -5
View File
@@ -24,7 +24,6 @@ import re
from collections import OrderedDict
import build
import config
import tarball
import util
@@ -32,8 +31,9 @@ import util
class FileManager(object):
"""Class to handle spec file %files section management."""
def __init__(self):
def __init__(self, config):
"""Set defaults for FileManager."""
self.config = config
self.packages = OrderedDict() # per sub-package file list for spec purposes
self.files = set() # global file set to weed out dupes
self.files_blacklist = set()
@@ -89,7 +89,7 @@ class FileManager(object):
def compat_exclude(self, filename):
"""Exclude non-library files if the package is for compatability."""
if not config.config_opts.get("compat"):
if not self.config.config_opts.get("compat"):
return False
patterns = [
@@ -228,8 +228,8 @@ class FileManager(object):
# if configured to do so, add .so files to the lib package instead of
# the dev package. THis is useful for packages with a plugin
# architecture like elfutils and mesa.
so_dest = 'lib' if config.config_opts.get('so_to_lib') else 'dev'
so_dest_ompi = 'openmpi' if config.config_opts.get('so_to_lib') else 'dev'
so_dest = 'lib' if self.config.config_opts.get('so_to_lib') else 'dev'
so_dest_ompi = 'openmpi' if self.config.config_opts.get('so_to_lib') else 'dev'
patterns = [
# Patterns for matching files, format is a tuple as follows:
autospec/git.py
+1 -2
View File
@@ -25,12 +25,11 @@ import subprocess
import build
import buildpattern
import config
import tarball
from util import call, write_out
def commit_to_git(path):
def commit_to_git(path, config):
"""Update package's git tree for autospec managed changes."""
call("git init", stdout=subprocess.DEVNULL, cwd=path)
autospec/license.py
+13 -11
View File
@@ -28,7 +28,6 @@ import sys
import urllib.parse
import chardet
import config
import download
import tarball
@@ -41,17 +40,17 @@ license_files = []
hashes = dict()
def process_licenses(lics):
def process_licenses(lics, translations, blacklist):
"""Handle licenses string from the license server.
The license server response may contain multiple space-separated licenses.
Add each license individually.
"""
for lic in lics.split():
add_license(lic)
add_license(lic, translations, blacklist)
def add_license(lic):
def add_license(lic, translations, blacklist):
"""Add licenses from the server.
Add license from license string lic after checking for duplication or
@@ -64,10 +63,10 @@ def add_license(lic):
result = False
# Translate the license if a translation exists
real_lic_str = config.license_translations.get(lic, lic)
real_lic_str = translations.get(lic, lic)
real_lics = real_lic_str.split()
for real_lic in real_lics:
if real_lic in config.license_blacklist:
if real_lic in blacklist:
continue
elif real_lic in licenses:
result = True
@@ -96,7 +95,7 @@ def decode_license(license):
return try_with_charset(license, chardet.detect(license)['encoding'])
def license_from_copying_hash(copying, srcdir):
def license_from_copying_hash(copying, srcdir, config):
"""Add licenses based on the hash of the copying file."""
try:
data = get_contents(copying)
@@ -124,7 +123,7 @@ def license_from_copying_hash(copying, srcdir):
page = response.decode('utf-8').strip()
if page:
print("License : ", page, " (server) (", hash_sum, ")")
process_licenses(page)
process_licenses(page, config.license_translations, config.license_blacklist)
if page != "none":
# Strip the build source directory off the front
@@ -139,7 +138,9 @@ def license_from_copying_hash(copying, srcdir):
return
if hash_sum in config.license_hashes:
add_license(config.license_hashes[hash_sum])
add_license(config.license_hashes[hash_sum],
config.license_translations,
config.license_blacklist)
else:
if not config.license_show:
return
@@ -148,7 +149,7 @@ def license_from_copying_hash(copying, srcdir):
print_warning("Visit {0} to enter".format(hash_url))
def scan_for_licenses(srcdir):
def scan_for_licenses(srcdir, config):
"""Scan the project directory for things we can use to guess a description and summary."""
targets = ["copyright",
"copyright.txt",
@@ -167,7 +168,8 @@ def scan_for_licenses(srcdir):
for dirpath, dirnames, files in os.walk(srcdir):
for name in files:
if name.lower() in targets or target_pat.search(name.lower()):
license_from_copying_hash(os.path.join(dirpath, name), srcdir)
license_from_copying_hash(os.path.join(dirpath, name),
srcdir, config)
if not licenses:
print_fatal(" Cannot find any license or a valid {}.license file!\n".format(tarball.name))
autospec/pkg_integrity.py
+20 -16
View File
@@ -210,6 +210,7 @@ class Verifier(object):
"""Set default values."""
self.url = kwargs.get('url', None)
self.package_sign_path = kwargs.get('package_sign_path', None)
self.config = kwargs.get('config', None)
print(SEPT)
@staticmethod
@@ -569,13 +570,13 @@ class GPGVerifier(Verifier):
EMAIL = get_email(pubkey_loc)
sign_status = verify_cli(pubkey_loc, self.package_path, self.package_sign_path)
if not sign_status:
if config.old_keyid:
compare_keys(KEYID_TRY, config.old_keyid)
if self.config.old_keyid:
compare_keys(KEYID_TRY, self.config.old_keyid)
self.print_result(self.package_path)
KEYID = KEYID_TRY
config.signature = self.key_url
config.config_opts['verify_required'] = True
config.rewrite_config_opts(os.path.dirname(self.package_path))
self.config.signature = self.key_url
self.config.config_opts['verify_required'] = True
self.config.rewrite_config_opts(os.path.dirname(self.package_path))
return True
else:
self.print_result(False, err_msg=sign_status.strerror)
@@ -833,14 +834,17 @@ def apply_verification(verifier, **kwargs):
return v.verify()
def from_disk(url, package_path, package_check, interactive=True):
def from_disk(url, package_path, package_check, config, interactive=True):
"""Run verification."""
verifier = get_verifier(package_path)
return apply_verification(verifier, **{
'package_path': package_path,
'package_check': package_check,
'url': url,
'interactive': interactive})
return apply_verification(verifier,
**{
'package_path': package_path,
'package_check': package_check,
'url': url,
'interactive': interactive,
'config': config,
})
def attempt_verification_per_domain(package_path, url):
@@ -879,7 +883,7 @@ def get_integrity_file(package_path):
return None
def check(url, download_path, interactive=True):
def check(url, download_path, config, interactive=True):
"""Run verification based on tar file url."""
package_name = filename_from_url(url)
package_path = os.path.join(download_path, package_name)
@@ -892,15 +896,15 @@ def check(url, download_path, interactive=True):
print_info('Performing package integrity verification')
verified = None
if package_check is not None:
verified = from_disk(url, package_path, package_check, interactive=interactive)
verified = from_disk(url, package_path, package_check, config, interactive=interactive)
elif package_path[-4:] == '.gem':
signature_file = get_signature_file(url, download_path)
verified = from_disk(url, package_path, signature_file, interactive=interactive)
verified = from_disk(url, package_path, signature_file, config, interactive=interactive)
else:
print_info('None of {}.(asc|sig|sign|sha256) is found in {}'.format(package_name, download_path))
signature_file = get_signature_file(url, download_path)
if signature_file is not None:
verified = from_disk(url, package_path, signature_file, interactive=interactive)
verified = from_disk(url, package_path, signature_file, config, interactive=interactive)
if verified is None:
print_info('Unable to find a signature')
verified = attempt_verification_per_domain(package_path, url)
@@ -936,7 +940,7 @@ def load_specfile(specfile):
def main(args):
"""Verify tar content with signature."""
from_disk(args.url, args.tar, args.sig)
from_disk(args.url, args.tar, args.sig, config.Config())
if __name__ == '__main__':
autospec/pkg_scan.py
+3 -4
View File
@@ -17,11 +17,10 @@
#
import subprocess
import config
import util
def get_whatrequires(pkg):
def get_whatrequires(pkg, yum_conf):
"""
Write list of packages.
@@ -30,7 +29,7 @@ def get_whatrequires(pkg):
"""
# clean up dnf cache to avoid 'no more mirrors repo' error
try:
subprocess.check_output(['dnf', '--config', config.yum_conf,
subprocess.check_output(['dnf', '--config', yum_conf,
'--releasever', 'clear', 'clean', 'all'])
except subprocess.CalledProcessError as err:
util.print_warning("Unable to clean dnf repo: {}, {}".format(pkg, err))
@@ -38,7 +37,7 @@ def get_whatrequires(pkg):
try:
out = subprocess.check_output(['dnf', 'repoquery',
'--config', config.yum_conf,
'--config', yum_conf,
'--releasever', 'clear',
'--archlist=src', '--recursive', '--queryformat=%{NAME}',
'--whatrequires', pkg]).decode('utf-8')
autospec/specdescription.py
+18 -19
View File
@@ -29,7 +29,6 @@
import os
import re
import config
import license
import util
@@ -84,7 +83,7 @@ def assign_description(description, score):
default_description_score = score
def description_from_spec(specfile):
def description_from_spec(specfile, translations, blacklist):
"""Parse any existing RPM specfiles."""
try:
with util.open_auto(specfile, 'r') as specfd:
@@ -105,15 +104,15 @@ def description_from_spec(specfile):
if line.startswith("License:") and not any(e in line for e in excludes):
splits = line.split(":")[1:]
words = ":".join(splits).strip()
if words in config.license_translations:
if words in translations:
print("Adding license from spec:", words)
license.add_license(words)
license.add_license(words, translations, blacklist)
else:
words = clean_license_string(words).split()
for word in words:
if ":" not in word and not word.startswith("@"):
print("Adding license from spec:", word)
license.add_license(word)
license.add_license(word, translations, blacklist)
if line.startswith("Summary: "):
assign_summary(line[9:], 4)
@@ -128,7 +127,7 @@ def description_from_spec(specfile):
assign_description(specdesc, 4)
def description_from_pkginfo(pkginfo):
def description_from_pkginfo(pkginfo, translations, blacklist):
"""Parse existing package info files."""
try:
with util.open_auto(pkginfo, 'r') as pkgfd:
@@ -146,15 +145,15 @@ def description_from_pkginfo(pkginfo):
if line.lower().startswith("license:") and not any(e in line for e in excludes):
splits = line.split(":")[1:]
words = ":".join(splits).strip()
if words in config.license_translations:
if words in translations:
print("Adding license from PKG-INFO:", words)
license.add_license(words)
license.add_license(words, translations, blacklist)
else:
words = clean_license_string(words).split()
for word in words:
if ":" not in word:
print("Adding license from PKG-INFO:", word)
license.add_license(word)
license.add_license(word, translations, blacklist)
for sub in ["Summary: ", "abstract: "]:
if line.startswith(sub):
@@ -241,7 +240,7 @@ def description_from_readme(readmefile):
assign_description(desc, score)
def scan_for_description(package, dirn):
def scan_for_description(package, dirn, translations, blacklist):
"""Scan the project directory for things we can use to guess a description and summary."""
test_pat = re.compile(r"tests?")
dirpath_seen = ""
@@ -253,13 +252,13 @@ def scan_for_description(package, dirn):
if name.lower().endswith(".pdf"):
continue
if name.lower().endswith(".spec"):
description_from_spec(os.path.join(dirpath, name))
description_from_spec(os.path.join(dirpath, name), translations, blacklist)
if name.lower().endswith("pkg-info"):
description_from_pkginfo(os.path.join(dirpath, name))
description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
if name.lower().endswith("meta.yml"):
description_from_pkginfo(os.path.join(dirpath, name))
description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
if name.lower().endswith("description"):
description_from_pkginfo(os.path.join(dirpath, name))
description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
if name.lower().endswith(".pc"):
summary_from_pkgconfig(os.path.join(dirpath, name), package)
if name.startswith("DESCRIPTION"):
@@ -272,13 +271,13 @@ def scan_for_description(package, dirn):
print("Summary :", default_summary.strip())
def load_specfile(specfile):
def load_specfile(specfile, description, summary):
"""Load specfile with parse results."""
if config.custom_desc:
specfile.default_desc = "\n".join(config.custom_desc)
if description:
specfile.default_desc = "\n".join(description)
else:
specfile.default_desc = default_description
if config.custom_summ:
specfile.default_sum = config.custom_summ[0]
if summary:
specfile.default_sum = summary[0]
else:
specfile.default_sum = default_summary
autospec/specfiles.py
+118 -120
View File
@@ -26,7 +26,6 @@ import types
from collections import OrderedDict
import buildreq
import config
import tarball
from util import _file_write
from util import open_auto
@@ -35,12 +34,13 @@ from util import open_auto
class Specfile(object):
"""Holds data and methods needed to write the spec file."""
def __init__(self, url, version, name, release):
def __init__(self, url, version, name, release, config):
"""Add default information for specfile template."""
self.url = url
self.version = version
self.name = name
self.release = release
self.config = config
self.keepstatic = False
self.urlban = ""
self.no_autostart = False
@@ -179,9 +179,9 @@ class Specfile(object):
self._write("Source{0} : {1}\n".format(count, source))
# if package is verified, include the signature in the source tarball
if self.keyid and config.signature:
if self.keyid and self.config.signature:
count += 1
self._write_strip(f"Source{count} : {config.signature}")
self._write_strip(f"Source{count} : {self.config.signature}")
for source in self.extra_sources:
count += 1
@@ -224,13 +224,13 @@ class Specfile(object):
def write_strip_command(self):
"""Write commands to prevent stripping binary if requested."""
if config.config_opts['nostrip']:
if self.config.config_opts['nostrip']:
self._write("# Suppress stripping binaries\n")
self._write("%define __strip /bin/true\n%define debug_package %{nil}\n")
def write_debug_command(self):
"""Write commands to prevent debug info generation if requested."""
if config.config_opts['nodebug']:
if self.config.config_opts['nodebug']:
self._write("# Suppress generation of debuginfo\n")
self._write("%global debug_package %{nil}\n")
@@ -243,8 +243,8 @@ class Specfile(object):
for count, patch in enumerate(self.patches):
self._write("Patch{0}: {1}\n".format(count + 1, patch.split()[0]))
# Write the version-specific patches
for version in config.verpatches:
for count, patch in enumerate(config.verpatches[version], start=count + 1):
for version in self.config.verpatches:
for count, patch in enumerate(self.config.verpatches[version], start=count + 1):
self._write("Patch{0}: {1}\n".format(count + 1, patch.split()[0]))
def write_description(self):
@@ -270,9 +270,9 @@ class Specfile(object):
deps["libexec"] = ["config", "license"]
deps["lib32"] = ["data", "license"]
deps["python"] = ["python3"]
if config.config_opts['dev_requires_extras']:
if self.config.config_opts.get('dev_requires_extras'):
deps["dev"].append("extras")
if config.config_opts['openmpi']:
if self.config.config_opts.get('openmpi'):
deps["dev"].append("openmpi")
for k, v in self.custom_extras.items():
if "requires" in v:
@@ -355,7 +355,7 @@ class Specfile(object):
if pkg in ["ignore", "main", "locales"]:
continue
for script in ["post", "pre"]:
content = config.read_conf_file("{}.{}".format(script, pkg))
content = self.config.read_conf_file("{}.{}".format(script, pkg))
if content:
self._write("\n%{0} {1}\n".format(script, pkg))
content = ['{}\n'.format(l) for l in content]
@@ -401,7 +401,7 @@ class Specfile(object):
if export_epoch:
# time.time() returns a float, but we only need second-precision
self._write_strip("export SOURCE_DATE_EPOCH={}".format(int(time.time())))
if config.config_opts['asneeded']:
if self.config.config_opts['asneeded']:
self._write_strip("unset LD_AS_NEEDED\n")
def write_proxy_exports(self):
@@ -418,9 +418,9 @@ class Specfile(object):
self._write_strip("{}\n".format(line))
self._write_strip("## make_prepend end")
if build32:
self._write_strip("make {} {} {}".format(config.parallel_build, self.extra_make, self.extra32_make))
self._write_strip("make {} {} {}".format(self.config.parallel_build, self.extra_make, self.extra32_make))
else:
self._write_strip("make {} {}".format(config.parallel_build, self.extra_make))
self._write_strip("make {} {}".format(self.config.parallel_build, self.extra_make))
def write_install_openmpi(self):
"""Write make install line (openmpi) to spec file."""
@@ -533,19 +533,19 @@ class Specfile(object):
destination))
self.apply_patches()
if self.default_pattern != 'cmake':
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ..")
self._write_strip("cp -a {} build32".format(self.tarball_prefix))
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("pushd ..")
self._write_strip("cp -a {} buildavx2".format(self.tarball_prefix))
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("pushd ..")
self._write_strip("cp -a {} buildavx512".format(self.tarball_prefix))
self._write_strip("popd")
if config.config_opts['openmpi']:
if self.config.config_opts['openmpi']:
self._write_strip("pushd ..")
self._write_strip("cp -a {} build-openmpi".format(self.tarball_prefix))
self._write_strip("popd")
@@ -570,7 +570,7 @@ class Specfile(object):
# a source of headaches for downstream users.
self._write_strip("export GCC_IGNORE_WERROR=1\n")
if config.config_opts['use_clang']:
if self.config.config_opts['use_clang']:
self._write_strip("export CC=clang\n")
self._write_strip("export CXX=clang++\n")
self._write_strip("export LD=ld.gold\n")
@@ -583,12 +583,12 @@ class Specfile(object):
if not self.set_gopath:
self._write_strip("export GOPROXY=file:///usr/share/goproxy")
if config.config_opts['optimize_size']:
if config.config_opts['use_clang']:
if self.config.config_opts['optimize_size']:
if self.config.config_opts['use_clang']:
flags.extend(["-Os", "-ffunction-sections", "-fdata-sections"])
else:
flags.extend(["-Os", "-ffunction-sections", "-fdata-sections", "-fno-semantic-interposition"])
if config.config_opts['security_sensitive']:
if self.config.config_opts['security_sensitive']:
flags.append("-fstack-protector-strong")
if arch == 'x86_64':
flags.append("-mzero-caller-saved-regs=used")
@@ -596,10 +596,10 @@ class Specfile(object):
flags.extend(["-O3", "-march=haswell"])
if self.need_avx512_flags:
flags.extend(["-O3", "-march=skylake-avx512"])
if config.config_opts['insecure_build']:
if self.config.config_opts['insecure_build']:
self._write_strip('export CFLAGS="-O3 -g -fopt-info-vec "\n')
self._write_strip("unset LDFLAGS\n")
if config.config_opts['conservative_flags']:
if self.config.config_opts['conservative_flags']:
self._write_strip('export CFLAGS="-O2 -g -Wp,-D_FORTIFY_SOURCE=2 '
"-fexceptions -fstack-protector "
"--param=ssp-buffer-size=32 -Wformat "
@@ -608,17 +608,17 @@ class Specfile(object):
'-march=westmere -mtune=haswell"\n')
self._write_strip("export CXXFLAGS=$CFLAGS\n")
self._write_strip("unset LDFLAGS\n")
if config.config_opts['use_clang']:
if self.config.config_opts['use_clang']:
self._write_strip("unset LDFLAGS\n")
if config.config_opts['funroll-loops']:
if config.config_opts['use_clang']:
if self.config.config_opts['funroll-loops']:
if self.config.config_opts['use_clang']:
flags.extend(["-O3"])
else:
flags.extend(["-O3", "-fno-semantic-interposition", "-falign-functions=32", "-fno-math-errno", "-fno-trapping-math"])
if self.default_pattern != 'qmake':
if config.config_opts['use_lto']:
if self.config.config_opts['use_lto']:
flags.extend(["-O3", lto, "-ffat-lto-objects"])
if config.config_opts['use_clang']:
if self.config.config_opts['use_clang']:
self._write_strip("export AR=llvm-ar\n")
self._write_strip("export RANLIB=llvm-ranlib\n")
self._write_strip("export NM=llvm-nm\n")
@@ -628,26 +628,26 @@ class Specfile(object):
self._write_strip("export NM=gcc-nm\n")
else:
flags.extend(["-fno-lto"])
if config.config_opts['fast-math']:
if self.config.config_opts['fast-math']:
flags.extend(["-ffast-math", "-ftree-loop-vectorize"])
if config.config_opts['pgo']:
if self.config.config_opts['pgo']:
flags.extend(["-O3"])
if self.gcov_file:
flags = list(filter((lto).__ne__, flags))
flags.extend(["-O3", "-fauto-profile=%{{SOURCE{0}}}".format(self.source_index[self.sources["gcov"][0]])])
if flags or config.config_opts['broken_c++']:
if flags or self.config.config_opts['broken_c++']:
flags = sorted(list(set(flags)))
self._write_strip('export CFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags)))
self._write_strip('export FCFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags)))
self._write_strip('export FFLAGS="$CFLAGS {0} "\n'.format(" ".join(flags)))
# leave the export CXXFLAGS line open in case
self._write('export CXXFLAGS="$CXXFLAGS {0} '.format(" ".join(flags)))
if config.config_opts['broken_c++']:
if self.config.config_opts['broken_c++']:
self._write('-std=gnu++98')
# close the open quote from CXXFLAGS export and add newline
self._write('"\n')
if config.profile_payload and config.profile_payload[0] and not self.need_avx2_flags:
if self.config.profile_payload and self.config.profile_payload[0] and not self.need_avx2_flags:
genflags = []
useflags = []
genflags.extend(["-fprofile-generate", "-fprofile-dir=/var/tmp/pgo", "-fprofile-update=atomic"])
@@ -667,7 +667,7 @@ class Specfile(object):
def write_check(self):
"""Write check section to spec file."""
if self.tests_config and not config.config_opts['skip_tests']:
if self.tests_config and not self.config.config_opts['skip_tests']:
self._write_strip("%check")
self._write_strip("export LANG=C.UTF-8")
self.write_proxy_exports()
@@ -685,7 +685,7 @@ class Specfile(object):
def write_profile_payload(self, pattern=None):
"""Write the profile_payload specified for this package."""
if not config.profile_payload:
if not self.config.profile_payload:
return
use_subdir = True
init = ""
@@ -694,20 +694,20 @@ class Specfile(object):
init = f"{self.get_profile_generate_flags()}" \
f"%configure " \
f"{self.disable_static} " \
f"{config.extra_configure} " \
f"{config.extra_configure64}"
f"{self.config.extra_configure} " \
f"{self.config.extra_configure64}"
elif pattern == "configure_ac":
init = f"{self.get_profile_generate_flags()}" \
f"%reconfigure " \
f"{self.disable_static} " \
f"{config.extra_configure} " \
f"{config.extra_configure64}"
f"{self.config.extra_configure} " \
f"{self.config.extra_configure64}"
elif pattern == "autogen":
init = f"{self.get_profile_generate_flags()}" \
f"%autogen " \
f"{self.disable_static} " \
f"{config.extra_configure} " \
f"{config.extra_configure64}"
f"{self.config.extra_configure} " \
f"{self.config.extra_configure64}"
elif pattern == "cmake":
use_subdir = False
init = f"{self.get_profile_generate_flags()}"
@@ -723,7 +723,7 @@ class Specfile(object):
if use_subdir and self.subdir:
self._write_strip("popd")
self._write_strip("\n")
self._write_strip("\n".join(config.profile_payload))
self._write_strip("\n".join(self.config.profile_payload))
self._write_strip("\nmake clean\n")
if post:
self._write_strip(post)
@@ -738,7 +738,7 @@ class Specfile(object):
self.write_license_files()
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ../build32/" + self.subdir)
self._write_strip("%make_install32 {} {}".format(self.extra_make_install,
self.extra_make32_install))
@@ -750,17 +750,17 @@ class Specfile(object):
self._write_strip("fi")
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("pushd ../buildavx512/" + self.subdir)
self._write_strip("%s_avx512 %s\n" % (self.install_macro, self.extra_make_install))
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("pushd ../buildavx2/" + self.subdir)
self._write_strip("%s_avx2 %s\n" % (self.install_macro, self.extra_make_install))
self._write_strip("popd")
if config.config_opts['openmpi']:
if self.config.config_opts['openmpi']:
self._write_strip("pushd ../build-openmpi/" + self.subdir)
self.write_install_openmpi()
self._write_strip("popd")
@@ -1004,7 +1004,7 @@ class Specfile(object):
if self.subdir:
self._write_strip("pushd " + self.subdir)
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd clr-build32")
self._write_strip("%make_install32 {} {}".format(self.extra_make_install,
self.extra_make32_install))
@@ -1016,17 +1016,17 @@ class Specfile(object):
self._write_strip("fi")
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("pushd clr-build-avx512")
self._write_strip("%s_avx512 %s || :\n" % (self.install_macro, self.extra_make_install))
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("pushd clr-build-avx2")
self._write_strip("%s_avx2 %s || :\n" % (self.install_macro, self.extra_make_install))
self._write_strip("popd")
if config.config_opts['openmpi']:
if self.config.config_opts['openmpi']:
self._write_strip("pushd clr-build-openmpi")
self.write_install_openmpi()
self._write_strip("popd")
@@ -1040,8 +1040,7 @@ class Specfile(object):
self.write_find_lang()
@staticmethod
def get_profile_generate_flags():
def get_profile_generate_flags(self):
"""Return profile generate flags if proper configuration is set.
If config.profile_payload is non-empty, returns
@@ -1053,7 +1052,7 @@ class Specfile(object):
otherwise an empty string is returned.
"""
if config.profile_payload and config.profile_payload[0]:
if self.config.profile_payload and self.config.profile_payload[0]:
return 'CFLAGS="${CFLAGS_GENERATE}" ' \
'CXXFLAGS="${CXXFLAGS_GENERATE}" ' \
'FFLAGS="${FFLAGS_GENERATE}" ' \
@@ -1061,8 +1060,7 @@ class Specfile(object):
'LDFLAGS="${LDFLAGS_GENERATE}" '
return ""
@staticmethod
def get_profile_use_flags():
def get_profile_use_flags(self):
"""Return profile generate flags if proper configuration is set.
If config.profile_payload is non-empty, returns
@@ -1074,7 +1072,7 @@ class Specfile(object):
otherwise an empty string is returned.
"""
if config.profile_payload and config.profile_payload[0]:
if self.config.profile_payload and self.config.profile_payload[0]:
return 'CFLAGS="${CFLAGS_USE}" ' \
'CXXFLAGS="${CXXFLAGS_USE}" ' \
'FFLAGS="${FFLAGS_USE}" ' \
@@ -1118,13 +1116,13 @@ class Specfile(object):
self._write_strip("{0}%configure {1} {2} {3}"
.format(self.get_profile_use_flags(),
self.disable_static,
config.extra_configure,
config.extra_configure64))
self.config.extra_configure,
self.config.extra_configure64))
self.write_make_line()
if self.subdir:
self._write_strip("popd")
self._write_strip("\n")
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ../build32/" + self.subdir)
self.write_build_prepend()
self.write_32bit_exports()
@@ -1134,12 +1132,12 @@ class Specfile(object):
"--host=i686-generic-linux-gnu "
"--target=i686-clr-linux-gnu"
.format(self.disable_static,
config.extra_configure,
config.extra_configure32))
self.config.extra_configure,
self.config.extra_configure32))
self.write_make_line(True)
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("unset PKG_CONFIG_PATH")
self._write_strip("pushd ../buildavx2/" + self.subdir)
self.write_build_prepend()
@@ -1148,12 +1146,12 @@ class Specfile(object):
self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"")
self._write_strip("%configure {0} {1} {2} "
.format(self.disable_static,
config.extra_configure,
config.extra_configure_avx2))
self.config.extra_configure,
self.config.extra_configure_avx2))
self.write_make_line()
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("unset PKG_CONFIG_PATH")
self._write_strip("pushd ../buildavx512/" + self.subdir)
self.write_build_prepend()
@@ -1162,12 +1160,12 @@ class Specfile(object):
self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=skylake-avx512\"")
self._write_strip("%configure {0} {1} {2} "
.format(self.disable_static,
config.extra_configure,
config.extra_configure_avx512))
self.config.extra_configure,
self.config.extra_configure_avx512))
self.write_make_line()
self._write_strip("popd")
if config.config_opts['openmpi']:
if self.config.config_opts['openmpi']:
self._write_strip("pushd ../build-openmpi/" + self.subdir)
self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh")
self._write_strip("module load openmpi")
@@ -1178,9 +1176,9 @@ class Specfile(object):
self._write_strip("export FFLAGS=\"$FFLAGS -m64 -march=haswell\"")
self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"")
self._write_strip("./configure {0} \\\n{1} {2}"
.format(config.conf_args_openmpi,
.format(self.config.conf_args_openmpi,
self.disable_static,
config.extra_configure_openmpi))
self.config.extra_configure_openmpi))
self.write_make_line()
self._write_strip("module unload openmpi")
self._write_strip("popd")
@@ -1199,12 +1197,12 @@ class Specfile(object):
self._write_strip("{0}%reconfigure {1} {2} {3}"
.format(self.get_profile_use_flags(),
self.disable_static,
config.extra_configure,
config.extra_configure64))
self.config.extra_configure,
self.config.extra_configure64))
self.write_make_line()
if self.subdir:
self._write_strip("popd")
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ../build32/" + self.subdir)
self.write_build_prepend()
self.write_32bit_exports()
@@ -1214,12 +1212,12 @@ class Specfile(object):
"--host=i686-generic-linux-gnu "
"--target=i686-clr-linux-gnu"
.format(self.disable_static,
config.extra_configure,
config.extra_configure32))
self.config.extra_configure,
self.config.extra_configure32))
self.write_make_line(True)
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("unset PKG_CONFIG_PATH")
self._write_strip("pushd ../buildavx2/" + self.subdir)
self.write_build_prepend()
@@ -1228,12 +1226,12 @@ class Specfile(object):
self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"")
self._write_strip("%reconfigure {0} {1} {2} "
.format(self.disable_static,
config.extra_configure,
config.extra_configure_avx2))
self.config.extra_configure,
self.config.extra_configure_avx2))
self.write_make_line()
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("unset PKG_CONFIG_PATH")
self._write_strip("pushd ../buildavx512/" + self.subdir)
self.write_build_prepend()
@@ -1242,8 +1240,8 @@ class Specfile(object):
self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=skylake-avx512\"")
self._write_strip("%reconfigure {0} {1} {2} "
.format(self.disable_static,
config.extra_configure,
config.extra_configure_avx512))
self.config.extra_configure,
self.config.extra_configure_avx512))
self.write_make_line()
self._write_strip("popd")
@@ -1263,12 +1261,12 @@ class Specfile(object):
if self.subdir:
self._write_strip("popd")
self._write_strip("\n")
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ../build32/" + self.subdir)
self.write_32bit_exports()
self.write_make_line(True)
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("pushd ../buildavx2" + self.subdir)
self.write_build_prepend()
self._write_strip("export CFLAGS=\"$CFLAGS -m64 -march=haswell\"")
@@ -1276,7 +1274,7 @@ class Specfile(object):
self._write_strip("export LDFLAGS=\"$LDFLAGS -m64 -march=haswell\"")
self.write_make_line()
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("pushd ../buildavx512" + self.subdir)
self.write_build_prepend()
self._write_strip("export CFLAGS=\"$CFLAGS -m64 -march=skylake-avx512 -mprefer-vector-width=512\"")
@@ -1298,11 +1296,11 @@ class Specfile(object):
self._write_strip("{0}%autogen {1} {2} {3}"
.format(self.get_profile_use_flags(),
self.disable_static,
config.extra_configure,
config.extra_configure64))
self.config.extra_configure,
self.config.extra_configure64))
self.write_make_line()
self._write_strip("\n")
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ../build32/" + self.subdir)
self.write_build_prepend()
self.write_32bit_exports()
@@ -1312,12 +1310,12 @@ class Specfile(object):
"--host=i686-generic-linux-gnu "
"--target=i686-clr-linux-gnu"
.format(self.disable_static,
config.extra_configure,
config.extra_configure32))
self.config.extra_configure,
self.config.extra_configure32))
self.write_make_line(True)
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("pushd ../buildavx2/" + self.subdir)
self.write_build_prepend()
self._write_strip('export CFLAGS="$CFLAGS -m64 -march=haswell "')
@@ -1325,12 +1323,12 @@ class Specfile(object):
self._write_strip('export LDFLAGS="$LDFLAGS -m64 -march=haswell "')
self._write_strip("%autogen {0} {1} {2} "
.format(self.disable_static,
config.extra_configure,
config.extra_configure_avx2))
self.config.extra_configure,
self.config.extra_configure_avx2))
self.write_make_line()
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("pushd ../buildavx512/" + self.subdir)
self.write_build_prepend()
self._write_strip('export CFLAGS="$CFLAGS -m64 -march=skylake-avx512 "')
@@ -1338,8 +1336,8 @@ class Specfile(object):
self._write_strip('export LDFLAGS="$LDFLAGS -m64 -march=skylake-avx512 "')
self._write_strip("%autogen {0} {1} {2} "
.format(self.disable_static,
config.extra_configure,
config.extra_configure_avx512))
self.config.extra_configure,
self.config.extra_configure_avx512))
self.write_make_line()
self._write_strip("popd")
@@ -1354,9 +1352,9 @@ class Specfile(object):
self._write_strip("export MAKEFLAGS=%{?_smp_mflags}")
if self.subdir:
self._write_strip("pushd " + self.subdir)
self._write_strip("python3 setup.py build " + config.extra_configure)
self._write_strip("python3 setup.py build " + self.config.extra_configure)
self._write_strip("\n")
if self.tests_config and not config.config_opts['skip_tests']:
if self.tests_config and not self.config.config_opts['skip_tests']:
self._write_strip("%check")
# Prevent setuptools from hitting the internet
self.write_proxy_exports()
@@ -1388,9 +1386,9 @@ class Specfile(object):
self.write_variables()
if self.subdir:
self._write_strip("pushd " + self.subdir)
self._write_strip("python3.6 setup.py build -b py3 " + config.extra_configure)
self._write_strip("python3.6 setup.py build -b py3 " + self.config.extra_configure)
self._write_strip("\n")
if self.tests_config and not config.config_opts['skip_tests']:
if self.tests_config and not self.config.config_opts['skip_tests']:
self._write_strip("%check")
# Prevent setuptools from hitting the internet
self.write_proxy_exports()
@@ -1535,7 +1533,7 @@ class Specfile(object):
self.write_make_line()
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("mkdir -p clr-build-avx2")
self._write_strip("pushd clr-build-avx2")
saved_avx2flags = self.need_avx2_flags
@@ -1549,7 +1547,7 @@ class Specfile(object):
self.write_make_line()
self._write_strip("popd")
if config.config_opts['use_avx512']:
if self.config.config_opts['use_avx512']:
self._write_strip("mkdir -p clr-build-avx512")
self._write_strip("pushd clr-build-avx512")
saved_avx512flags = self.need_avx512_flags
@@ -1563,7 +1561,7 @@ class Specfile(object):
self.write_make_line()
self._write_strip("popd")
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("mkdir -p clr-build32")
self._write_strip("pushd clr-build32")
self.write_build_prepend()
@@ -1577,7 +1575,7 @@ class Specfile(object):
self._write_strip("unset PKG_CONFIG_PATH")
self._write_strip("popd")
if config.config_opts['openmpi']:
if self.config.config_opts['openmpi']:
self._write_strip("mkdir -p clr-build-openmpi")
self._write_strip("pushd clr-build-openmpi")
self._write_strip(". /usr/share/defaults/etc/profile.d/modules.sh")
@@ -1607,9 +1605,9 @@ class Specfile(object):
def write_qmake_pattern(self):
"""Write qmake build pattern to spec file."""
extra_qmake_args = ""
if config.config_opts['use_clang']:
if self.config.config_opts['use_clang']:
extra_qmake_args = "-spec linux-clang "
if config.config_opts['use_lto']:
if self.config.config_opts['use_lto']:
extra_qmake_args += "-config ltcg -config fat-static-lto "
else:
extra_qmake_args += "QMAKE_CFLAGS+=-fno-lto QMAKE_CXXFLAGS+=-fno-lto "
@@ -1624,18 +1622,18 @@ class Specfile(object):
if self.subdir:
self._write_strip("pushd " + self.subdir)
self._write_strip("%qmake {} {}".format(extra_qmake_args, config.extra_configure))
self._write_strip("%qmake {} {}".format(extra_qmake_args, self.config.extra_configure))
self._write_strip("test -r config.log && cat config.log")
self.write_make_line()
if self.subdir:
self._write_strip("popd")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip("pushd ../buildavx2/" + self.subdir)
self._write("%qmake 'QT_CPU_FEATURES.x86_64 += avx avx2 bmi bmi2 f16c fma lzcnt popcnt'\\\n")
self._write(" QMAKE_CFLAGS+=-march=haswell QMAKE_CXXFLAGS+=-march=haswell \\\n")
self._write(" QMAKE_LFLAGS+=-march=haswell {} {}\n".format(extra_qmake_args, config.extra_configure))
self._write(" QMAKE_LFLAGS+=-march=haswell {} {}\n".format(extra_qmake_args, self.config.extra_configure))
self.write_make_line()
self._write_strip("popd")
@@ -1708,7 +1706,7 @@ class Specfile(object):
self.write_proxy_exports()
self._write_strip("export LANG=C.UTF-8")
self.write_variables()
self._write_strip("scons{} {}".format(config.parallel_build, config.extra_configure))
self._write_strip("scons{} {}".format(self.config.parallel_build, self.config.extra_configure))
self.write_build_append()
self._write_strip("\n")
self._write_strip("%install")
@@ -1853,23 +1851,23 @@ class Specfile(object):
if self.subdir:
self._write_strip("pushd " + self.subdir)
self._write_strip('CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" LDFLAGS="$LDFLAGS" meson --libdir=lib64 --prefix=/usr --buildtype=plain {0} {1} builddir'
.format(config.extra_configure,
config.extra_configure64))
.format(self.config.extra_configure,
self.config.extra_configure64))
self._write_strip("ninja -v -C builddir")
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip('CFLAGS="$CFLAGS -m64 -march=haswell" CXXFLAGS="$CXXFLAGS -m64 -march=haswell " LDFLAGS="$LDFLAGS -m64 -march=haswell" '
'meson --libdir=lib64/haswell --prefix=/usr --buildtype=plain {0} '
'{1} builddiravx2'.format(config.extra_configure, config.extra_configure64))
'{1} builddiravx2'.format(self.config.extra_configure, self.config.extra_configure64))
self._write_strip('ninja -v -C builddiravx2')
if self.subdir:
self._write_strip("popd")
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip("pushd ../build32/" + self.subdir)
self.write_32bit_exports()
self._write_strip('meson '
'--libdir=lib32 --prefix=/usr --buildtype=plain {0} {1} builddir'
.format(config.extra_configure,
config.extra_configure32))
.format(self.config.extra_configure,
self.config.extra_configure32))
self._write_strip('ninja -v -C builddir')
self._write_strip('popd')
@@ -1879,7 +1877,7 @@ class Specfile(object):
self._write_strip("%install")
self.write_install_prepend()
self.write_license_files()
if config.config_opts['32bit']:
if self.config.config_opts['32bit']:
self._write_strip('pushd ../build32/' + self.subdir)
self._write_strip('DESTDIR=%{buildroot} ninja -C builddir install')
self._write_strip("if [ -d %{buildroot}/usr/lib32/pkgconfig ]")
@@ -1891,7 +1889,7 @@ class Specfile(object):
self._write_strip("popd")
if self.subdir:
self._write_strip("pushd " + self.subdir)
if config.config_opts['use_avx2']:
if self.config.config_opts['use_avx2']:
self._write_strip('DESTDIR=%{buildroot} ninja -C builddiravx2 install')
self._write_strip("DESTDIR=%{buildroot} ninja -C builddir install")
@@ -1951,7 +1949,7 @@ class Specfile(object):
# Write version-specific patch commands
for version in self.verpatches:
if self.verpatches[version]:
self._write("cd ../{}\n".format(self.build_dirs[config.versions[version]]))
self._write("cd ../{}\n".format(self.build_dirs[self.config.versions[version]]))
for p in self.verpatches[version]:
name = p.split(None, 1)[0]
if name == p:
autospec/tarball.py
+11 -12
View File
@@ -27,7 +27,6 @@ from collections import OrderedDict
import build
import buildpattern
import buildreq
import config
import download
from util import call, do_regex, get_sha1sum, print_fatal, write_out
@@ -266,7 +265,7 @@ def detect_build_from_url(url):
buildpattern.set_build_pattern("phpize", 10)
def set_multi_version(ver):
def set_multi_version(ver, config):
"""Add ver to multi_version set and return latest version."""
global multi_version
@@ -287,7 +286,7 @@ def set_multi_version(ver):
return latest
def name_and_version(name_arg, version_arg, filemanager):
def name_and_version(name_arg, version_arg, filemanager, config):
"""Parse the url for the package name and version."""
global rawname
global url
@@ -307,7 +306,7 @@ def name_and_version(name_arg, version_arg, filemanager):
# rawname == name in this case
name = name_arg
rawname = name_arg
version = set_multi_version(version_arg)
version = set_multi_version(version_arg, config)
return name, rawname, convert_version(version, name)
name = name_arg
@@ -439,7 +438,7 @@ def name_and_version(name_arg, version_arg, filemanager):
# override name and version from commandline
name = name_arg if name_arg else name
version = version_arg if version_arg else version
version = set_multi_version(version)
version = set_multi_version(version, config)
return name, rawname, convert_version(version, name)
@@ -474,7 +473,7 @@ def process_go_archives(go_archives):
buildpattern.sources["godep"] += [url_info, url_mod, url_zip]
def process_multiver_archives(main_src, multiver_archives):
def process_multiver_archives(main_src, multiver_archives, config):
"""Set up multiversion archives."""
config_versions = config.parse_config_versions(build.download_path)
# Check if exist more than one version.
@@ -485,10 +484,10 @@ def process_multiver_archives(main_src, multiver_archives):
buildpattern.sources["version"].append(extraurl)
multiver_archives.append(extraurl)
multiver_archives.append('')
set_multi_version(None)
set_multi_version(None, config)
def process_archives(main_src, archives):
def process_archives(main_src, archives, config):
"""Process extra sources needed by package.
This sources include: archives, go archives and multiversion.
@@ -502,7 +501,7 @@ def process_archives(main_src, archives):
process_go_archives(go_archives)
# Add multiversion for the rest of the patterns
else:
process_multiver_archives(main_src, multiver_archives)
process_multiver_archives(main_src, multiver_archives, config)
full_archives = archives + go_archives + multiver_archives
# Download and extract full list
@@ -534,7 +533,7 @@ def extract_sources(main_src, archives_src):
src.extract()
def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager):
def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager, config):
"""Download and process the tarball at url_arg."""
global url
global name
@@ -552,7 +551,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager):
# Create the download path for content and set build.download_path
create_download_path(target)
# determine name and version of package
name, rawname, version = name_and_version(name_arg, ver_arg, filemanager)
name, rawname, version = name_and_version(name_arg, ver_arg, filemanager, config)
# Store the top-level version
config.versions[version] = url
# set gcov file information, must be done after name is set since the gcov
@@ -570,7 +569,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager):
print_header()
# Download and process extra sources: archives, go archives and
# multiversion
archives_src = process_archives(main_src, archives)
archives_src = process_archives(main_src, archives, config)
# Extract all sources
extract_sources(main_src, archives_src)
tests/test_build.py
+59 -39
View File
@@ -3,18 +3,12 @@ import tempfile
import os
from unittest.mock import patch, mock_open, MagicMock
import build
import config
import files
class TestBuildpattern(unittest.TestCase):
@classmethod
def setUpClass(self):
"""
Class setup method to configure necessary modules
"""
build.config.setup_patterns()
def setUp(self):
"""
Test setup method to reset the buildpattern module
@@ -25,7 +19,6 @@ class TestBuildpattern(unittest.TestCase):
build.base_path = None
build.download_path = None
build.buildreq.buildreqs = set()
build.config.config_opts['32bit'] = False
def test_setup_workingdir(self):
"""
@@ -40,10 +33,10 @@ class TestBuildpattern(unittest.TestCase):
"""
Test simple_pattern_pkgconfig with match
"""
build.buildreq.config.config_opts['32bit'] = False
build.simple_pattern_pkgconfig('line to test for testpkg.xyz',
r'testpkg.xyz',
'testpkg')
'testpkg',
False)
self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs)
self.assertEqual(build.must_restart, 1)
@@ -51,10 +44,10 @@ class TestBuildpattern(unittest.TestCase):
"""
Test simple_pattern_pkgconfig with match and 32bit option set
"""
build.buildreq.config.config_opts['32bit'] = True
build.simple_pattern_pkgconfig('line to test for testpkg.zyx',
r'testpkg.zyx',
'testpkgz')
'testpkgz',
True)
self.assertIn('pkgconfig(32testpkgz)', build.buildreq.buildreqs)
self.assertIn('pkgconfig(testpkgz)', build.buildreq.buildreqs)
self.assertEqual(build.must_restart, 1)
@@ -65,7 +58,8 @@ class TestBuildpattern(unittest.TestCase):
"""
build.simple_pattern_pkgconfig('line to test for somepkg.xyz',
r'testpkg.xyz',
'testpkg')
'testpkg',
False)
self.assertEqual(build.buildreq.buildreqs, set())
self.assertEqual(build.must_restart, 0)
@@ -95,7 +89,8 @@ class TestBuildpattern(unittest.TestCase):
"""
Test failed_pattern with no match
"""
build.failed_pattern('line to test for failure: somepkg', r'(test)', 0)
conf = config.Config()
build.failed_pattern('line to test for failure: somepkg', conf, r'(test)', 0)
self.assertEqual(build.buildreq.buildreqs, set())
self.assertEqual(build.must_restart, 0)
@@ -104,7 +99,8 @@ class TestBuildpattern(unittest.TestCase):
Test failed_pattern with buildtool unset and initial match, but no
match in failed_commands.
"""
build.failed_pattern('line to test for failure: testpkg', r'(test)', 0)
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg', conf, r'(test)', 0)
self.assertEqual(build.buildreq.buildreqs, set())
self.assertEqual(build.must_restart, 0)
@@ -112,7 +108,9 @@ class TestBuildpattern(unittest.TestCase):
"""
Test failed_pattern with buildtool unset and match in failed_commands
"""
build.failed_pattern('line to test for failure: lex', r'(lex)', 0)
conf = config.Config()
conf.setup_patterns()
build.failed_pattern('line to test for failure: lex', conf, r'(lex)', 0)
self.assertIn('flex', build.buildreq.buildreqs)
self.assertEqual(build.must_restart, 1)
@@ -120,8 +118,9 @@ class TestBuildpattern(unittest.TestCase):
"""
Test failed_pattern with buildtool set to pkgconfig
"""
build.buildreq.config.config_opts['32bit'] = False
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg.xyz',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='pkgconfig')
@@ -132,7 +131,10 @@ class TestBuildpattern(unittest.TestCase):
"""
Test failed_pattern with buildtool set to R
"""
conf = config.Config()
conf.setup_patterns()
build.failed_pattern('line to test for failure: testpkg.r',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='R')
@@ -144,7 +146,9 @@ class TestBuildpattern(unittest.TestCase):
"""
Test failed_pattern with buildtool set to perl
"""
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg.pl',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='perl')
@@ -155,7 +159,9 @@ class TestBuildpattern(unittest.TestCase):
"""
Test failed_pattern with buildtool set to pypi
"""
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg.py',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='pypi')
@@ -167,7 +173,9 @@ class TestBuildpattern(unittest.TestCase):
Test failed_pattern with buildtool set to ruby, but no match in
config.gems, it should just prepend 'rubygem-' to the package name.
"""
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg.rb',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='ruby')
@@ -180,7 +188,10 @@ class TestBuildpattern(unittest.TestCase):
config.gems. In the particular case of test/unit, the result should
be rubygem-test-unit.
"""
conf = config.Config()
conf.setup_patterns()
build.failed_pattern('line to test for failure: test/unit',
conf,
r'(test/unit)',
0, # verbose=0
buildtool='ruby')
@@ -192,7 +203,10 @@ class TestBuildpattern(unittest.TestCase):
Test failed_pattern with buildtool set to ruby table and a match in
config.gems
"""
conf = config.Config()
conf.setup_patterns()
build.failed_pattern('line to test for failure: test/unit',
conf,
r'(test/unit)',
0, # verbose=0
buildtool='ruby table')
@@ -204,7 +218,9 @@ class TestBuildpattern(unittest.TestCase):
Test failed_pattern with buildtool set to ruby table but no match in
config.gems. This should not modify anything.
"""
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='ruby table')
@@ -216,7 +232,9 @@ class TestBuildpattern(unittest.TestCase):
Test failed_pattern with buildtool set to maven, but no match in
config.maven_jars, it should just prepend 'mvn-' to the package name.
"""
conf = config.Config()
build.failed_pattern('line to test for failure: testpkg',
conf,
r'(testpkg)',
0, # verbose=0
buildtool='maven')
@@ -229,7 +247,10 @@ class TestBuildpattern(unittest.TestCase):
config.maven_jars. In the particular case of aether, the corresponding
maven jar is 'mvn-aether-core'
"""
conf = config.Config()
conf.setup_patterns()
build.failed_pattern('line to test for failure: aether',
conf,
r'(aether)',
0, # verbose=0
buildtool='maven')
@@ -244,8 +265,6 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
build.config.config_opts['32bit'] = True
call_backup = build.util.call
build.util.call = mock_util_call
@@ -269,8 +288,6 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
build.config.config_opts['32bit'] = True
call_backup = build.util.call
build.util.call = mock_util_call
@@ -295,8 +312,6 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
build.config.config_opts['32bit'] = True
call_backup = build.util.call
build.util.call = mock_util_call
@@ -320,18 +335,19 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
build.config.config_opts['32bit'] = True
conf = config.Config()
conf.setup_patterns()
conf.config_opts['32bit'] = True
call_backup = build.util.call
build.util.call = mock_util_call
fm = files.FileManager()
fm = files.FileManager(conf)
open_name = 'build.util.open_auto'
content = 'line 1\nwhich: no qmake\nexiting'
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
build.parse_build_results('testname', 0, fm)
build.parse_build_results('testname', 0, fm, conf)
build.util.call = call_backup
@@ -347,17 +363,18 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
conf = config.Config()
conf.setup_patterns()
call_backup = build.util.call
build.util.call = mock_util_call
fm = files.FileManager()
fm = files.FileManager(conf)
open_name = 'build.util.open_auto'
content = 'line 1\nchecking for Apache test module support\nexiting'
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
build.parse_build_results('testname', 0, fm)
build.parse_build_results('testname', 0, fm, conf)
build.util.call = call_backup
@@ -369,11 +386,12 @@ class TestBuildpattern(unittest.TestCase):
Test parse_build_results with a test log indicating failure due to a
missing package.
"""
build.config.setup_patterns()
conf = config.Config()
conf.setup_patterns()
call_backup = build.util.call
open_auto_backup = build.util.open_auto
build.util.call = MagicMock(return_value=None)
fm = files.FileManager()
fm = files.FileManager(conf)
with open('tests/builderrors', 'r') as f:
builderrors = f.readlines()
@@ -382,7 +400,7 @@ class TestBuildpattern(unittest.TestCase):
input, output = error.strip('\n').split('|')
build.buildreq.buildreqs = set()
build.util.open_auto = mock_open(read_data=input)
build.parse_build_results('testname', 0, fm)
build.parse_build_results('testname', 0, fm, conf)
self.assertIn(output, build.buildreq.buildreqs)
self.assertGreater(build.must_restart, 0)
@@ -398,10 +416,11 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
conf = config.Config()
conf.setup_patterns()
call_backup = build.util.call
build.util.call = mock_util_call
fm = files.FileManager()
fm = files.FileManager(conf)
open_name = 'build.util.open_auto'
content = 'line 1\n' \
@@ -414,7 +433,7 @@ class TestBuildpattern(unittest.TestCase):
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
build.parse_build_results('testname', 0, fm)
build.parse_build_results('testname', 0, fm, conf)
build.util.call = call_backup
@@ -432,10 +451,11 @@ class TestBuildpattern(unittest.TestCase):
def mock_util_call(cmd):
del cmd
build.config.setup_patterns()
conf = config.Config()
conf.setup_patterns()
call_backup = build.util.call
build.util.call = mock_util_call
fm = files.FileManager()
fm = files.FileManager(conf)
open_name = 'build.util.open_auto'
content = 'line 1\n' \
@@ -450,7 +470,7 @@ class TestBuildpattern(unittest.TestCase):
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
build.parse_build_results('testname', 0, fm)
build.parse_build_results('testname', 0, fm, conf)
build.util.call = call_backup
tests/test_buildreq.py
+53 -54
View File
@@ -5,6 +5,7 @@ from unittest.mock import MagicMock, mock_open, patch
import json
import io
import buildreq
import config
class TestBuildreq(unittest.TestCase):
@@ -15,7 +16,6 @@ class TestBuildreq(unittest.TestCase):
Class setup method to configure necessary modules
"""
buildreq.banned_buildreqs.add('bannedreq')
buildreq.config.setup_patterns()
def setUp(self):
"""
@@ -26,9 +26,6 @@ class TestBuildreq(unittest.TestCase):
buildreq.requires = set()
buildreq.verbose = False
buildreq.cargo_bin = False
buildreq.config.config_opts['32bit'] = False
buildreq.config.cmake_modules = {}
buildreq.config.os_packages = set()
buildreq.buildpattern.pattern_strength = 0
def test_add_buildreq(self):
@@ -53,29 +50,28 @@ class TestBuildreq(unittest.TestCase):
buildreqs but not yet present in requires
"""
buildreq.add_buildreq('testreq')
self.assertTrue(buildreq.add_requires('testreq'))
self.assertTrue(buildreq.add_requires('testreq', ['testreq']))
self.assertIn('testreq', buildreq.requires)
def test_add_requires_not_in_buildreqs(self):
"""
Test add_requires with unbanned new req not present in buildreqs.
"""
self.assertFalse(buildreq.add_requires('testreq'))
self.assertFalse(buildreq.add_requires('testreq', []))
self.assertNotIn('testreq', buildreq.requires)
def test_add_pkgconfig_buildreq(self):
"""
Test add_pkgconfig_buildreq with config_opts['32bit'] set to False
"""
self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq'))
self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', False))
self.assertIn('pkgconfig(testreq)', buildreq.buildreqs)
def test_add_pkgconfig_buildreq_32bit(self):
"""
Test add_pkgconfig_buildreq with config_opts['32bit'] set to True
"""
buildreq.config.config_opts['32bit'] = True
self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq'))
self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', True))
self.assertIn('pkgconfig(testreq)', buildreq.buildreqs)
self.assertIn('pkgconfig(32testreq)', buildreq.buildreqs)
@@ -83,14 +79,14 @@ class TestBuildreq(unittest.TestCase):
"""
Test configure_ac_line with standard pattern
"""
buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])')
buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False)
self.assertIn('ncurses-devel', buildreq.buildreqs)
def test_configure_ac_line_comment(self):
"""
Test configure_ac_line with commented line
"""
buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])')
buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False)
self.assertEqual(buildreq.buildreqs, set())
def test_configure_ac_line_pkg_check_modules(self):
@@ -101,7 +97,7 @@ class TestBuildreq(unittest.TestCase):
buildreq.configure_ac_line(
'PKG_CHECK_MODULES(prefix, '
'[module > 2 module2 < 2], '
'action-if-found, action-if-not-found)')
'action-if-found, action-if-not-found)', False)
self.assertEqual(buildreq.buildreqs,
set(['pkgconfig(module)', 'pkgconfig(module2)']))
@@ -112,7 +108,7 @@ class TestBuildreq(unittest.TestCase):
buildreq.configure_ac_line(
'XDT_CHECK_PACKAGE(prefix, '
'[module = 2 module2 > 9], '
'action-if-found, action-if-not-found)')
'action-if-found, action-if-not-found)', False)
self.assertEqual(buildreq.buildreqs,
set(['pkgconfig(module)', 'pkgconfig(module2)']))
@@ -122,7 +118,7 @@ class TestBuildreq(unittest.TestCase):
"""
buildreq.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], '
'action-if-found, '
'action-if-not-found)')
'action-if-not-found)', False)
self.assertEqual(buildreq.buildreqs,
set(['pkgconfig(module1)', 'pkgconfig(module2)']))
@@ -143,7 +139,7 @@ class TestBuildreq(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'fname'), 'w') as f:
f.write(content)
buildreq.parse_configure_ac(os.path.join(tmpd, 'fname'))
buildreq.parse_configure_ac(os.path.join(tmpd, 'fname'), False)
self.assertEqual(buildreq.buildpattern.default_pattern, 'configure_ac')
self.assertEqual(buildreq.buildreqs,
@@ -205,7 +201,7 @@ class TestBuildreq(unittest.TestCase):
content = 'does not matter, let us mock'
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.parse_cargo_toml('filename')
buildreq.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3'])
buildreq.os.path.exists = exists_backup
buildreq.toml.loads = loads_backup
@@ -249,11 +245,13 @@ class TestBuildreq(unittest.TestCase):
"""
Test rakefile parsing with both configured gems and unconfigured gems
"""
conf = config.Config()
conf.setup_patterns()
open_name = 'buildreq.util.open_auto'
content = "line1\nrequire 'bundler/gem_tasks'\nline3\nrequire 'nope'"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.rakefile('filename')
buildreq.rakefile('filename', conf.gems)
self.assertEqual(buildreq.buildreqs, set(['rubygem-rubygems-tasks']))
@@ -291,7 +289,7 @@ class TestBuildreq(unittest.TestCase):
'req7 == 3.3.3\n'
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.grab_python_requirements('filename')
buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3'])
self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7']))
@@ -307,7 +305,7 @@ class TestBuildreq(unittest.TestCase):
' req7 == 3.3.3\n '
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.grab_python_requirements('filename')
buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3'])
self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7']))
@@ -321,7 +319,7 @@ class TestBuildreq(unittest.TestCase):
"setup_requires=['req2']"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.add_setup_py_requires('filename')
buildreq.add_setup_py_requires('filename', ['req1', 'req2'])
self.assertEqual(buildreq.buildreqs, set(['req1', 'req2']))
self.assertEqual(buildreq.requires, set(['req1']))
@@ -336,7 +334,7 @@ class TestBuildreq(unittest.TestCase):
"'req3']\n"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.add_setup_py_requires('filename')
buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3'])
self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3']))
self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3']))
@@ -354,7 +352,7 @@ class TestBuildreq(unittest.TestCase):
"]\n"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.add_setup_py_requires('filename')
buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3'])
self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3']))
self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3']))
@@ -372,7 +370,7 @@ class TestBuildreq(unittest.TestCase):
"]\n"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.add_setup_py_requires('filename')
buildreq.add_setup_py_requires('filename', ['req1', 'req2'])
self.assertEqual(buildreq.buildreqs, set(['req1', 'req2']))
self.assertEqual(buildreq.requires, set(['req1', 'req2']))
@@ -385,7 +383,7 @@ class TestBuildreq(unittest.TestCase):
content = "install_requires=[reqname, 'req1', 'req2']\n"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.add_setup_py_requires('filename')
buildreq.add_setup_py_requires('filename', ['req1', 'req2'])
self.assertEqual(buildreq.buildreqs, set(['req1', 'req2']))
self.assertEqual(buildreq.requires, set(['req1', 'req2']))
@@ -398,7 +396,7 @@ class TestBuildreq(unittest.TestCase):
content = "install_requires=reqname"
m_open = mock_open(read_data=content)
with patch(open_name, m_open, create=True):
buildreq.add_setup_py_requires('filename')
buildreq.add_setup_py_requires('filename', [])
self.assertEqual(buildreq.buildreqs, set())
self.assertEqual(buildreq.requires, set())
@@ -425,6 +423,7 @@ class TestBuildreq(unittest.TestCase):
much to test here that uses the same logic, a representative test
should be sufficient.
"""
conf = config.Config()
with tempfile.TemporaryDirectory() as tmpd:
os.mkdir(os.path.join(tmpd, 'subdir'))
open(os.path.join(tmpd, 'subdir', 'test.go'), 'w').close()
@@ -433,7 +432,7 @@ class TestBuildreq(unittest.TestCase):
open(os.path.join(tmpd, 'SConstruct'), 'w').close()
open(os.path.join(tmpd, 'meson.build'), 'w').close()
buildreq.scan_for_configure(tmpd, "", "")
buildreq.scan_for_configure(tmpd, "", "", conf)
self.assertEqual(buildreq.buildreqs,
set(['buildreq-golang', 'buildreq-cmake', 'buildreq-scons', 'buildreq-distutils3', 'buildreq-meson']))
@@ -443,6 +442,7 @@ class TestBuildreq(unittest.TestCase):
Test scan_for_configure when distutils is being used for the build
pattern to test pypi metadata handling.
"""
conf = config.Config()
orig_summary = buildreq.specdescription.default_summary
orig_sscore = buildreq.specdescription.default_summary_score
orig_pypi_name = buildreq.pypidata.get_pypi_name
@@ -459,7 +459,7 @@ class TestBuildreq(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmpd:
os.mkdir(os.path.join(tmpd, 'subdir'))
open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd)
buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf)
ssummary = buildreq.specdescription.default_summary
buildreq.specdescription.default_summary = orig_summary
@@ -476,6 +476,7 @@ class TestBuildreq(unittest.TestCase):
Test scan_for_configure when distutils is being used for the build
pattern to test pypi metadata file override handling.
"""
conf = config.Config()
open_name = 'buildreq.open'
orig_summary = buildreq.specdescription.default_summary
orig_sscore = buildreq.specdescription.default_summary_score
@@ -491,7 +492,7 @@ class TestBuildreq(unittest.TestCase):
open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
open(os.path.join(tmpd, 'pypi.json'), 'w').close()
with patch(open_name, m_open, create=True):
buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd)
buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf)
ssummary = buildreq.specdescription.default_summary
buildreq.specdescription.default_summary = orig_summary
@@ -506,11 +507,13 @@ class TestBuildreq(unittest.TestCase):
Test parse_cmake to ensure accurate detection of versioned and
unversioned pkgconfig modules.
"""
conf = config.Config()
conf.setup_patterns()
content = 'pkg_check_modules(GLIB gio-unix-2.0>=2.46.0 glib-2.0 REQUIRED)'
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'fname'), 'w') as f:
f.write(content)
buildreq.parse_cmake(os.path.join(tmpd, 'fname'))
buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
self.assertEqual(buildreq.buildreqs,
set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)']))
@@ -520,11 +523,13 @@ class TestBuildreq(unittest.TestCase):
Test parse_cmake to ensure accurate handling of versioned
pkgconfig modules with whitespace.
"""
conf = config.Config()
conf.setup_patterns()
content = 'pkg_check_modules(GLIB gio-unix-2.0 >= 2.46.0 glib-2.0 REQUIRED)'
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'fname'), 'w') as f:
f.write(content)
buildreq.parse_cmake(os.path.join(tmpd, 'fname'))
buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
self.assertEqual(buildreq.buildreqs,
set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)']))
@@ -533,6 +538,8 @@ class TestBuildreq(unittest.TestCase):
"""
Test parse_cmake to ensure it ignores pkg_check_modules in comments.
"""
conf = config.Config()
conf.setup_patterns()
content = '''
# For example, consider the following patch to some CMakeLists.txt.
# - pkg_check_modules(FOO REQUIRED foo>=1.0)
@@ -541,7 +548,7 @@ class TestBuildreq(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'fname'), 'w') as f:
f.write(content)
buildreq.parse_cmake(os.path.join(tmpd, 'fname'))
buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
self.assertEqual(buildreq.buildreqs,
set([]))
@@ -551,11 +558,13 @@ class TestBuildreq(unittest.TestCase):
Test parse_cmake to ensure accurate handling of versioned
pkgconfig modules with variable version strings.
"""
conf = config.Config()
conf.setup_patterns()
content = 'pkg_check_modules(AVCODEC libavcodec${_avcodec_ver} libavutil$_avutil_ver)'
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'fname'), 'w') as f:
f.write(content)
buildreq.parse_cmake(os.path.join(tmpd, 'fname'))
buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
self.assertEqual(buildreq.buildreqs,
set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)']))
@@ -564,7 +573,7 @@ class TestBuildreq(unittest.TestCase):
"""
Test parse_cmake to ensure accurate handling of find_package.
"""
buildreq.config.cmake_modules = {
cmake_modules = {
"valid": "valid",
"valid_but_commented": "valid_but_commented",
"different_name": "another_name",
@@ -578,7 +587,7 @@ find_package(different_name)
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'fname'), 'w') as f:
f.write(content)
buildreq.parse_cmake(os.path.join(tmpd, 'fname'))
buildreq.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False)
self.assertEqual(buildreq.buildreqs,
set(['valid', 'another_name']))
@@ -679,44 +688,37 @@ find_package(different_name)
result = buildreq._get_desc_field("Field2", "\n".join(lines))
self.assertEqual(result, [])
@patch('buildreq.config.os_packages')
def test_parse_r_desc_depends(self, os_pkgs):
def test_parse_r_desc_depends(self):
"""Test parsing of R description Depends field."""
pkgs = ['R-pkg1']
os_pkgs.__contains__.side_effect = lambda val: val in pkgs
open_name = 'buildreq.util.open_auto'
content = 'Depends: pkg1'
m_open = mock_open(read_data=content)
with patch(open_name, m_open):
buildreq.parse_r_description('filename')
buildreq.parse_r_description('filename', pkgs)
self.assertTrue('R-pkg1' in buildreq.buildreqs)
@patch('buildreq.config.os_packages')
def test_parse_r_desc_imports(self, os_pkgs):
def test_parse_r_desc_imports(self):
"""Test parsing of an R description Imports field."""
pkgs = ['R-pkg2']
os_pkgs.__contains__.side_effect = lambda val: val in pkgs
open_name = 'buildreq.util.open_auto'
content = 'Imports: pkg2'
m_open = mock_open(read_data=content)
with patch(open_name, m_open):
buildreq.parse_r_description('filename')
buildreq.parse_r_description('filename', pkgs)
self.assertTrue('R-pkg2' in buildreq.buildreqs)
@patch('buildreq.config.os_packages')
def test_parse_r_desc_linkingto(self, os_pkgs):
def test_parse_r_desc_linkingto(self):
"""Test parsing of an R description LinkingTo field."""
pkgs = ['R-pkg3']
os_pkgs.__contains__.side_effect = lambda val: val in pkgs
open_name = 'buildreq.util.open_auto'
content = 'LinkingTo: pkg3'
m_open = mock_open(read_data=content)
with patch(open_name, m_open):
buildreq.parse_r_description('filename')
buildreq.parse_r_description('filename', pkgs)
self.assertTrue('R-pkg3' in buildreq.buildreqs)
@patch('buildreq.config.os_packages')
def test_parse_r_desc_multiple(self, os_pkgs):
def test_parse_r_desc_multiple(self):
"""Test parsing of an R description file that captures multiple fields."""
pkgs = [
'R-pkg1',
@@ -724,7 +726,6 @@ find_package(different_name)
'R-pkg3',
'R-pkg4',
]
os_pkgs.__contains__.side_effect = lambda val: val in pkgs
open_name = 'buildreq.util.open_auto'
content = [
'Field1: foo',
@@ -735,7 +736,7 @@ find_package(different_name)
]
m_open = mock_open(read_data='\n'.join(content))
with patch(open_name, m_open):
buildreq.parse_r_description('filename')
buildreq.parse_r_description('filename', pkgs)
self.assertFalse('R-foo' in buildreq.buildreqs)
self.assertFalse('R-bar' in buildreq.buildreqs)
self.assertTrue('R-pkg1' in buildreq.buildreqs)
@@ -743,13 +744,11 @@ find_package(different_name)
self.assertTrue('R-pkg3' in buildreq.buildreqs)
self.assertTrue('R-pkg4' in buildreq.buildreqs)
@patch('buildreq.config.os_packages')
def test_parse_r_desc_not_in_os(self, os_pkgs):
def test_parse_r_desc_not_in_os(self):
"""Test parsing of an R description file with some non-OS packages."""
pkgs = [
'R-pkg1',
]
os_pkgs.__contains__.side_effect = lambda val: val in pkgs
open_name = 'buildreq.util.open_auto'
content = [
'Imports: pkg1, pkg2',
@@ -757,7 +756,7 @@ find_package(different_name)
]
m_open = mock_open(read_data='\n'.join(content))
with patch(open_name, m_open):
buildreq.parse_r_description('filename')
buildreq.parse_r_description('filename', pkgs)
self.assertTrue('R-pkg1' in buildreq.buildreqs)
self.assertFalse('R-pkg2' in buildreq.buildreqs)
self.assertFalse('R-pkg3' in buildreq.buildreqs)
tests/test_check.py
+17 -16
View File
@@ -4,7 +4,7 @@ import unittest
from unittest.mock import mock_open, patch
import check
import config
def mock_generator(rv=None):
def mock_f(*args, **kwargs):
@@ -19,12 +19,6 @@ class TestTest(unittest.TestCase):
@classmethod
def setUpClass(self):
self.open_name = 'check.util.open_auto'
check.config.config_opts['skip_tests'] = False
check.config.config_opts['allow_test_failures'] = False
check.config.config_opts['32bit'] = False
check.config.config_opts['use_avx2'] = False
check.config.config_opts['use_avx512'] = False
check.config.config_opts['openmpi'] = False
check.os.path.isfile = mock_generator(True)
@classmethod
@@ -49,7 +43,7 @@ class TestTest(unittest.TestCase):
m_open = mock_open()
open_name = 'util.open'
with patch(open_name, m_open, create=True):
check.check_regression('pkgdir')
check.check_regression('pkgdir', False)
check.count.parse_log = parse_log_backup
@@ -72,7 +66,7 @@ class TestTest(unittest.TestCase):
m_open = mock_open()
open_name = 'util.open'
with patch(open_name, m_open, create=True):
check.check_regression('pkgdir')
check.check_regression('pkgdir', False)
check.count.parse_log = parse_log_backup
@@ -94,13 +88,14 @@ class TestTest(unittest.TestCase):
"""
Test scan_for_tests with makecheck suite
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['Makefile.in'])
content = 'check:'
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
check.buildpattern.default_pattern = "configure"
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
check.buildpattern.default_pattern = "make"
@@ -111,12 +106,13 @@ class TestTest(unittest.TestCase):
"""
Test scan_for_tests with makecheck suite via Makefile.am
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['Makefile.am'])
m_open = mock_open()
with patch(self.open_name, m_open, create=True):
check.buildpattern.default_pattern = "configure_ac"
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
check.buildpattern.default_pattern = "make"
@@ -127,10 +123,11 @@ class TestTest(unittest.TestCase):
"""
Test scan_for_tests with perlcheck suite
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['Makefile.PL'])
check.buildpattern.default_pattern = "cpan"
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
check.buildpattern.default_pattern = "make"
self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test')
@@ -139,13 +136,14 @@ class TestTest(unittest.TestCase):
"""
Test scan_for_tests with perlcheck suite via Makefile.in
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['Makefile.in'])
content = 'test:'
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
check.buildpattern.default_pattern = "cpan"
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
check.buildpattern.default_pattern = "make"
@@ -155,13 +153,14 @@ class TestTest(unittest.TestCase):
"""
Test scan_for_tests with setup.py suite
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['setup.py'])
content = 'test_suite'
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
check.buildpattern.default_pattern = "distutils3"
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
check.buildpattern.default_pattern = "make"
@@ -173,13 +172,14 @@ class TestTest(unittest.TestCase):
"""
Test scan_for_tests with cmake suite
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['CMakeLists.txt'])
content = 'enable_testing'
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
check.buildpattern.default_pattern = "cmake"
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
check.buildpattern.default_pattern = "make"
@@ -191,9 +191,10 @@ class TestTest(unittest.TestCase):
Test scan_for_tests with tox.ini in the files list, should add several
build requirements
"""
conf = config.Config()
listdir_backup = os.listdir
check.os.listdir = mock_generator(['tox.ini'])
check.scan_for_tests('pkgdir')
check.scan_for_tests('pkgdir', conf)
check.os.listdir = listdir_backup
self.assertEqual(check.buildreq.buildreqs,
set(['tox',
tests/test_commitmessage.py
+21 -18
View File
@@ -3,6 +3,7 @@ import unittest.mock as mock
import os
import tempfile
import commitmessage
import config
class TestCommitmessage(unittest.TestCase):
@@ -10,7 +11,6 @@ class TestCommitmessage(unittest.TestCase):
def setUp(self):
commitmessage.tarball.name = 'testball'
commitmessage.tarball.version = '0.0.1'
commitmessage.config.old_version = '0.0.0'
self.workingdir = tempfile.TemporaryDirectory()
commitmessage.build.setup_workingdir(self.workingdir.name)
@@ -59,7 +59,7 @@ class TestCommitmessage(unittest.TestCase):
# last items
expected_msg = [""] + GOOD_NEWS.split('\n')[3:13]
expected_cvs = set()
self.assertEqual(commitmessage.process_NEWS('NEWS'),
self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'),
(expected_msg, expected_cvs))
def test_process_NEWS_bad_news(self):
@@ -71,7 +71,7 @@ class TestCommitmessage(unittest.TestCase):
with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
# make GOOD_NEWS irrelevant by replacing current version
newsfile.write(GOOD_NEWS.replace('0.0.1', '0.0.0'))
self.assertEqual(commitmessage.process_NEWS('NEWS'), ([], set()))
self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'), ([], set()))
def test_process_NEWS_good_cves(self):
"""
@@ -90,7 +90,7 @@ class TestCommitmessage(unittest.TestCase):
.replace('change2.2', 'CVE-2-2')\
.split('\n')[3:13]
expected_cvs = set(['CVE-2-1', 'CVE-2-2'])
self.assertEqual(commitmessage.process_NEWS('NEWS'),
self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'),
(expected_msg, expected_cvs))
def test_process_NEWS_long(self):
@@ -111,7 +111,7 @@ class TestCommitmessage(unittest.TestCase):
expected_msg.extend(['1', '2', '3', '4', '5', '6', '7', '',
'(NEWS truncated at 15 lines)', ''])
expected_cvs = set()
self.assertEqual(commitmessage.process_NEWS('NEWS'),
self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0'),
(expected_msg, expected_cvs))
def test_guess_commit_message(self):
@@ -119,9 +119,11 @@ class TestCommitmessage(unittest.TestCase):
Test guess_commit_message() with mocked internal functions and both
commitmessage information and cves available from newsfile.
"""
conf = config.Config()
conf.old_version = "0.0.0"
process_NEWS_backup = commitmessage.process_NEWS
def mock_process_NEWS(newsfile):
def mock_process_NEWS(newsfile, old_version):
return (['', 'commit', 'message', 'with', 'cves', ''],
set(['cve1', 'cve2']))
@@ -129,7 +131,7 @@ class TestCommitmessage(unittest.TestCase):
open_name = 'util.open_auto'
with mock.patch(open_name, create=True) as mock_open:
mock_open.return_value = mock.MagicMock()
commitmessage.guess_commit_message("")
commitmessage.guess_commit_message("", conf)
# reset mocks before asserting so a failure doesn't cascade to
# other tests
commitmessage.process_NEWS = process_NEWS_backup
@@ -147,23 +149,23 @@ class TestCommitmessage(unittest.TestCase):
also available from config, which changes the first line of the commmit
message.
"""
conf = config.Config()
process_NEWS_backup = commitmessage.process_NEWS
def mock_process_NEWS(newsfile):
def mock_process_NEWS(newsfile, old_version):
return (['', 'commit', 'message', 'with', 'cves', ''],
set(['cve1', 'cve2']))
commitmessage.process_NEWS = mock_process_NEWS
commitmessage.config.cves = set(['CVE-1234-5678'])
commitmessage.config.old_version = None # Allow cve title to be set
conf.cves = set(['CVE-1234-5678'])
conf.old_version = None # Allow cve title to be set
open_name = 'util.open_auto'
with mock.patch(open_name, create=True) as mock_open:
mock_open.return_value = mock.MagicMock()
commitmessage.guess_commit_message("")
commitmessage.guess_commit_message("", conf)
# reset mocks before asserting so a failure doesn't cascade to
# other tests
commitmessage.process_NEWS = process_NEWS_backup
commitmessage.config.cves = set()
fh = mock_open.return_value.__enter__.return_value
fh.write.assert_called_with(
'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n'
@@ -178,23 +180,23 @@ class TestCommitmessage(unittest.TestCase):
message. Additionally there is imported key info that will be displayed
at the end of the message.
"""
conf = config.Config()
process_NEWS_backup = commitmessage.process_NEWS
def mock_process_NEWS(newsfile):
def mock_process_NEWS(newsfile, old_version):
return (['', 'commit', 'message', 'with', 'cves', ''],
set(['cve1', 'cve2']))
commitmessage.process_NEWS = mock_process_NEWS
commitmessage.config.cves = set(['CVE-1234-5678'])
commitmessage.config.old_version = None # Allow cve title to be set
conf.cves = set(['CVE-1234-5678'])
conf.old_version = None # Allow cve title to be set
open_name = 'util.open_auto'
with mock.patch(open_name, create=True) as mock_open:
mock_open.return_value = mock.MagicMock()
commitmessage.guess_commit_message("keyinfo content")
commitmessage.guess_commit_message("keyinfo content", conf)
# reset mocks before asserting so a failure doesn't cascade to
# other tests
commitmessage.process_NEWS = process_NEWS_backup
commitmessage.config.cves = set()
fh = mock_open.return_value.__enter__.return_value
fh.write.assert_called_with(
'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n'
@@ -206,12 +208,13 @@ class TestCommitmessage(unittest.TestCase):
"""
Tests scan_for_changes using temporary directories
"""
conf = config.Config()
with tempfile.TemporaryDirectory() as tmpd:
with open(os.path.join(tmpd, 'changelog.txt'), 'w') as newsfile:
newsfile.write('new changelog file')
with tempfile.TemporaryDirectory() as tmpd1:
commitmessage.scan_for_changes(tmpd1, tmpd)
commitmessage.scan_for_changes(tmpd1, tmpd, conf.transforms)
self.assertTrue(os.path.isfile(tmpd1 + '/ChangeLog'))
tests/test_files.py
+5 -4
View File
@@ -20,7 +20,8 @@ def mock_return(retval):
class TestFiles(unittest.TestCase):
def setUp(self):
self.fm = FileManager()
conf = config.Config()
self.fm = FileManager(conf)
def test_banned_path(self):
"""
@@ -66,14 +67,14 @@ class TestFiles(unittest.TestCase):
"""
Test compat_exclude with a file that shouldn't be excluded.
"""
config.config_opts['compat'] = True
self.fm.config.config_opts['compat'] = True
self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so.1'))
def test_compat_exclude_exclude_file(self):
"""
Test compat_exclude with a file that should be excluded.
"""
config.config_opts['compat'] = True
self.fm.config.config_opts['compat'] = True
self.assertTrue(self.fm.compat_exclude('/usr/lib64/libfoo.so'))
def test_compat_exclude_not_compat_mode(self):
@@ -81,7 +82,7 @@ class TestFiles(unittest.TestCase):
Test compat_exclude with a file that should be excluded but isn't
because the package isn't being run in compat mode.
"""
config.config_opts['compat'] = False
self.fm.config.config_opts['compat'] = False
self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so'))
def test_file_pat_match(self):
tests/test_infile_update_spec.py
+2 -1
View File
@@ -1,5 +1,6 @@
import unittest
import config
import infile_update_spec
import specfiles
@@ -8,7 +9,7 @@ class TestUpdateSpecfile(unittest.TestCase):
def setUp(self):
# url, version, name, release
url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz"
self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1')
self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config())
self.bb_dict = {
"DEPENDS": "ncurses gettext-native",
tests/test_license.py
+32 -22
View File
@@ -7,6 +7,7 @@ from unittest.mock import patch, mock_open, MagicMock
import pycurl
import config
import download
import license
import util
@@ -14,20 +15,17 @@ import util
class TestLicense(unittest.TestCase):
@classmethod
def setUpClass(self):
license.config.setup_patterns()
def setUp(self):
license.licenses = []
license.config.license_fetch = None
def test_add_license(self):
"""
Test add_license from valid string, Apache-2 should be translated to
Apache-2.0
"""
self.assertTrue(license.add_license('Apache-2'))
conf = config.Config()
conf.setup_patterns()
self.assertTrue(license.add_license('Apache-2', conf.license_translations, conf.license_blacklist))
self.assertIn('Apache-2.0', license.licenses)
def test_add_license_present(self):
@@ -36,8 +34,10 @@ class TestLicense(unittest.TestCase):
the licenses list. Should return True and should not modify the
licenses list. GPL-3 translates to GPL-3.0.
"""
conf = config.Config()
conf.setup_patterns()
license.licenses.append('GPL-3.0')
self.assertTrue(license.add_license('GPL-3'))
self.assertTrue(license.add_license('GPL-3', conf.license_translations, conf.license_blacklist))
self.assertEqual(['GPL-3.0'], license.licenses)
def test_add_license_blacklisted(self):
@@ -45,18 +45,22 @@ class TestLicense(unittest.TestCase):
Test add_license from string in license_blacklist. Should return False
and should not modify the licenses list.
"""
conf = config.Config()
conf.setup_patterns()
# sanity check to make sure the licenses list is empty before the later
# assertIn() call
self.assertEqual(license.licenses, [])
self.assertFalse(license.add_license('License'))
self.assertFalse(license.add_license('License', conf.license_translations, conf.license_blacklist))
self.assertNotIn('License', license.licenses)
def test_license_from_copying_hash(self):
"""
Test license_from_copying_hash with valid license file
"""
license.license_from_copying_hash('tests/COPYING_TEST', '')
conf = config.Config()
conf.setup_patterns()
license.license_from_copying_hash('tests/COPYING_TEST', '', conf)
self.assertIn('GPL-3.0', license.licenses)
def test_license_from_copying_hash_no_license_show(self):
@@ -64,26 +68,26 @@ class TestLicense(unittest.TestCase):
Test license_from_copying_hash with invalid hash and no license_show
set
"""
bkup_hash = license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')]
conf = config.Config()
conf.setup_patterns()
# remove the hash from license_hashes
del(license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')])
license.config.license_show = "license.show.url"
license.license_from_copying_hash('tests/COPYING_TEST', '')
del(conf.license_hashes[license.get_sha1sum('tests/COPYING_TEST')])
conf.license_show = "license.show.url"
license.license_from_copying_hash('tests/COPYING_TEST', '', conf)
# restore the hash
license.config.license_hashes[license.get_sha1sum('tests/COPYING_TEST')] = bkup_hash
self.assertEquals(license.licenses, [])
def test_license_from_copying_hash_bad_license(self):
"""
Test license_from_copying_hash with invalid license file
"""
conf = config.Config()
content = util.get_contents("tests/COPYING_TEST").replace(b"GNU", b"SNU")
m_open = MagicMock()
m_open.__str__.return_value = content
with patch('license.get_contents', m_open, create=True):
license.license_from_copying_hash('copying.txt', '')
license.license_from_copying_hash('copying.txt', '', conf)
self.assertEquals(license.licenses, [])
@@ -114,13 +118,14 @@ class TestLicense(unittest.TestCase):
# set the mock curl
download.pycurl.Curl = MockCurl
license.config.license_fetch = 'license.server.url'
conf = config.Config()
conf.license_fetch = 'license.server.url'
# let's check that the proper thing is being printed as well
out = StringIO()
with redirect_stdout(out):
with self.assertRaises(SystemExit):
license.license_from_copying_hash('tests/COPYING_TEST', '')
license.license_from_copying_hash('tests/COPYING_TEST', '', conf)
self.assertIn('Unable to fetch license.server.url: Test Exception', out.getvalue())
@@ -167,12 +172,13 @@ class TestLicense(unittest.TestCase):
# set the mock curl
download.pycurl.Curl = MockCurl
license.config.license_fetch = 'license.server.url'
conf = config.Config()
conf.license_fetch = 'license.server.url'
# let's check that the proper thing is being printed as well
out = StringIO()
with redirect_stdout(out):
license.license_from_copying_hash('tests/COPYING_TEST', '')
license.license_from_copying_hash('tests/COPYING_TEST', '', conf)
self.assertIn('GPL-3.0', license.licenses)
self.assertIn('License : GPL-3.0 (server)', out.getvalue())
@@ -187,6 +193,8 @@ class TestLicense(unittest.TestCase):
"""
Test scan_for_licenses in temporary directory with valid license file
"""
conf = config.Config()
conf.setup_patterns()
with open('tests/COPYING_TEST', 'rb') as copyingf:
content = copyingf.read()
@@ -198,7 +206,7 @@ class TestLicense(unittest.TestCase):
for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
with open(os.path.join(tmpd, testf), 'w') as newtestf:
newtestf.write('test content')
license.scan_for_licenses(tmpd)
license.scan_for_licenses(tmpd, conf)
self.assertIn('GPL-3.0', license.licenses)
@@ -208,6 +216,8 @@ class TestLicense(unittest.TestCase):
Should not add any licenses, should print a fatal message, should exit
with a status code of 1.
"""
conf = config.Config()
conf.setup_patterns()
with tempfile.TemporaryDirectory() as tmpd:
# create some cruft for testing
for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
@@ -217,7 +227,7 @@ class TestLicense(unittest.TestCase):
out = StringIO()
with redirect_stdout(out):
with self.assertRaises(SystemExit) as thread:
license.scan_for_licenses(tmpd)
license.scan_for_licenses(tmpd, conf)
self.assertEqual(thread.exception.code, 1)
self.assertIn("Cannot find any license", out.getvalue())
tests/test_pkg_integrity.py
+60 -34
View File
@@ -6,6 +6,7 @@ import tempfile
import unittest
from unittest.mock import patch
import config
import download
import pkg_integrity
@@ -46,27 +47,30 @@ def mock_download_do_curl(url, dst=None):
@patch('download.do_curl', mock_download_do_curl)
class TestCheckFn(unittest.TestCase):
def setUp(self):
def mock_rewrite(path):
del path
pkg_integrity.config.rewrite_config_opts = mock_rewrite
pkg_integrity.config.config_opts['verify_required'] = False
def _mock_rewrite(self, path):
del path
def test_check_matching_sign_url(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
pkey = "023A4420C7EC6914.pkey"
shutil.copy(os.path.join(TESTKEYDIR, pkey), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd, conf)
self.assertTrue(result)
def test_check_with_existing_sign(self):
""" Download signature for local verification """
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd)
result = pkg_integrity.check(NOSIGN_PKT_URL, tmpd)
result = pkg_integrity.check(NOSIGN_PKT_URL, tmpd, conf)
self.assertTrue(result)
@@ -130,11 +134,8 @@ class TestDomainBasedVerifiers(unittest.TestCase):
@patch('download.do_curl', mock_download_do_curl)
class TestGEMShaVerifier(unittest.TestCase):
def setUp(self):
def mock_rewrite(path):
del path
pkg_integrity.config.rewrite_config_opts = mock_rewrite
pkg_integrity.config.config_opts['verify_required'] = False
def _mock_rewrite(self, path):
del path
def _mock_get_gem_info(pkg):
info = '''
@@ -153,67 +154,85 @@ class TestGEMShaVerifier(unittest.TestCase):
@patch('pkg_integrity.GEMShaVerifier.get_rubygems_info', _mock_get_gem_info)
def test_from_url(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
filen = os.path.basename(GEM_PKT)
shutil.copy(os.path.join(TESTDIR, filen), tmpd)
result = pkg_integrity.check(GEM_PKT, tmpd)
result = pkg_integrity.check(GEM_PKT, tmpd, conf)
self.assertTrue(result)
@patch('pkg_integrity.GEMShaVerifier.get_rubygems_info', _mock_get_gem_info)
def test_non_matchingsha(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
out_file = os.path.join(tmpd, os.path.basename(GEM_PKT))
f = open(out_file, 'wb')
f.write(b'this is made up data that will force a failure')
f.close()
with self.assertRaises(SystemExit) as a:
pkg_integrity.check(GEM_PKT, tmpd)
pkg_integrity.check(GEM_PKT, tmpd, conf)
self.assertEqual(a.exception.code, 1)
@patch('download.do_curl', mock_download_do_curl)
class TestGPGVerifier(unittest.TestCase):
def setUp(self):
def mock_rewrite(path):
del path
pkg_integrity.config.rewrite_config_opts = mock_rewrite
pkg_integrity.config.config_opts['verify_required'] = False
def _mock_rewrite(self, path):
del path
def test_from_url(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd, conf)
self.assertTrue(result)
def test_invalid_key(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd)
with open(os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD) + ".asc"), 'w') as ofile:
ofile.write("Invalid signature")
result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd)
result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd, conf)
self.assertIsNone(result)
def test_key_not_found(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd)
result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd)
result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd, conf)
self.assertIsNone(result)
def test_from_disk(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
out_key = out_file + ".asc"
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd)
result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key)
result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key, conf)
self.assertTrue(result)
def test_non_matchingsig(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -221,19 +240,26 @@ class TestGPGVerifier(unittest.TestCase):
f.write(b'made up date that will fail check')
f.close()
with self.assertRaises(SystemExit) as a:
pkg_integrity.check(PACKAGE_URL, tmpd)
pkg_integrity.check(PACKAGE_URL, tmpd, conf)
self.assertEqual(a.exception.code, 1)
def test_result_on_non_existent_pkg_path(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz',
'NonExistentPKG.tar.gz',
'NonExistentKey.asc')
'NonExistentKey.asc',
conf)
self.assertIsNone(result)
def test_result_on_nosign_package(self):
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd)
result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd)
result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd, conf)
self.assertIsNone(result)
@patch.object(pkg_integrity.GPGCli, 'exec_cmd')
@@ -270,10 +296,13 @@ class TestGPGVerifier(unittest.TestCase):
mock_exec.return_value = (b'', b'', 0)
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd, conf)
self.assertTrue(result)
self.assertEqual(mock_parse.call_count, 4)
self.assertEqual(mock_exec.call_count, 3)
@@ -308,11 +337,14 @@ class TestGPGVerifier(unittest.TestCase):
mock_exec.return_value = (b'', b'', 0)
conf = config.Config()
conf.rewrite_config_opts = self._mock_rewrite
conf.config_opts['verify_required'] = False
with tempfile.TemporaryDirectory() as tmpd:
shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
with self.assertRaises(SystemExit) as msg:
result = pkg_integrity.check(PACKAGE_URL, tmpd)
result = pkg_integrity.check(PACKAGE_URL, tmpd, conf)
self.assertEqual(msg.exception.code, 1)
self.assertEqual(mock_parse.call_count, 4)
self.assertEqual(mock_exec.call_count, 2)
@@ -335,12 +367,6 @@ class TestInputGetter(unittest.TestCase):
class TestUtils(unittest.TestCase):
def setUp(self):
def mock_rewrite(path):
del path
pkg_integrity.config.rewrite_config_opts = mock_rewrite
pkg_integrity.config.config_opts['verify_required'] = False
def test_get_verifier(self):
x = pkg_integrity.get_verifier('file.abcd')
self.assertEqual(x, None)
tests/test_specdescription.py
+5 -5
View File
@@ -1,5 +1,6 @@
import unittest
from unittest.mock import mock_open, patch
import config
import specdescription
@@ -15,7 +16,6 @@ class TestSpecdescription(unittest.TestCase):
specdescription.default_summary = "No detailed summary available"
specdescription.default_summary_score = 0
specdescription.license.licenses = []
specdescription.config.license_translations = {}
def test_clean_license_string(self):
"""
@@ -48,7 +48,7 @@ class TestSpecdescription(unittest.TestCase):
"%donewithdesc\n"
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
specdescription.description_from_spec("filename")
specdescription.description_from_spec("filename", {}, [])
self.assertEqual(specdescription.default_description,
"Here is the description section\n"
@@ -68,7 +68,7 @@ class TestSpecdescription(unittest.TestCase):
content = "# this is a specfile without much in it\n"
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
specdescription.description_from_spec("filename")
specdescription.description_from_spec("filename", {}, [])
self.assertEqual(specdescription.default_description,
"No detailed description available")
@@ -94,7 +94,7 @@ class TestSpecdescription(unittest.TestCase):
"donewithdesc:\n"
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
specdescription.description_from_pkginfo("filename")
specdescription.description_from_pkginfo("filename", {}, [])
self.assertEqual(specdescription.default_description,
"Here is the description section\n"
@@ -113,7 +113,7 @@ class TestSpecdescription(unittest.TestCase):
content = "# this is a pkginfo file without much in it\n"
m_open = mock_open(read_data=content)
with patch(self.open_name, m_open, create=True):
specdescription.description_from_pkginfo("filename")
specdescription.description_from_pkginfo("filename", {}, [])
self.assertEqual(specdescription.default_description,
"No detailed description available")
tests/test_specfile.py
+7 -14
View File
@@ -1,5 +1,6 @@
import unittest
import unittest.mock
import config
import specfiles
@@ -11,9 +12,10 @@ class TestSpecfileWrite(unittest.TestCase):
self.WRITES = []
def setUp(self):
conf = config.Config()
conf.config_opts['dev_requires_extras'] = False
url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz"
self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2')
specfiles.config.config_opts['dev_requires_extras'] = False
self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf)
def mock_write(string):
self.WRITES.append(string)
@@ -286,8 +288,6 @@ class TestSpecfileWrite(unittest.TestCase):
"""
test write_scriplets base test
"""
backup_read_conf_file = specfiles.config.read_conf_file
def mock_read_conf_file(name):
prefix = "pre" if "pre" in name else "post"
return ["{}-script line 1\n".format(prefix),
@@ -299,12 +299,11 @@ class TestSpecfileWrite(unittest.TestCase):
def writelines(stringlist):
self.WRITES.extend(stringlist)
specfiles.config.read_conf_file = mock_read_conf_file
self.specfile.config.read_conf_file = mock_read_conf_file
self.specfile.specfile = MockSpecfile()
self.specfile.packages["ruby"] = ["rubyfile1"]
self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
self.specfile.write_scriplets()
specfiles.config.read_conf_file = backup_read_conf_file
expect = ["\n%post ruby\n",
"post-script line 1\n\n",
"post-script line 2\n\n",
@@ -320,8 +319,6 @@ class TestSpecfileWrite(unittest.TestCase):
"""
test write_scriplets with only pre-scripts present.
"""
backup_read_conf_file = specfiles.config.read_conf_file
def mock_read_conf_file(name):
if "pre" in name:
return ["pre-script line 1\n",
@@ -335,12 +332,11 @@ class TestSpecfileWrite(unittest.TestCase):
def writelines(stringlist):
self.WRITES.extend(stringlist)
specfiles.config.read_conf_file = mock_read_conf_file
self.specfile.config.read_conf_file = mock_read_conf_file
self.specfile.specfile = MockSpecfile()
self.specfile.packages["ruby"] = ["rubyfile1"]
self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
self.specfile.write_scriplets()
specfiles.config.read_conf_file = backup_read_conf_file
expect = ["\n%pre ruby\n",
"pre-script line 1\n\n",
"pre-script line 2\n\n",
@@ -352,8 +348,6 @@ class TestSpecfileWrite(unittest.TestCase):
"""
test write_scriplets with no scripts present.
"""
backup_read_conf_file = specfiles.config.read_conf_file
def mock_read_conf_file(name):
return []
@@ -362,12 +356,11 @@ class TestSpecfileWrite(unittest.TestCase):
def writelines(stringlist):
self.WRITES.extend(stringlist)
specfiles.config.read_conf_file = mock_read_conf_file
self.specfile.config.read_conf_file = mock_read_conf_file
self.specfile.specfile = MockSpecfile()
self.specfile.packages["ruby"] = ["rubyfile1"]
self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
self.specfile.write_scriplets()
specfiles.config.read_conf_file = backup_read_conf_file
self.assertEqual([], self.WRITES)
def test_write_files_base(self):
tests/test_tarball.py
+7 -5
View File
@@ -1,5 +1,6 @@
import unittest
from unittest.mock import Mock, patch
import config
import tarball
@@ -123,11 +124,12 @@ def detect_build_test_generator(url, build_pattern):
def name_and_version_test_generator(url, name, version):
"""Create test for tarball.name_and_version method."""
@patch('tarball.config.parse_config_versions', Mock(return_value={}))
def generator(self):
"""Test template."""
conf = config.Config()
conf.parse_config_versions = Mock(return_value={})
tarball.url = url
n, _, v = tarball.name_and_version('', '', Mock())
n, _, v = tarball.name_and_version('', '', Mock(), conf)
self.assertEqual(name, n)
self.assertEqual(version, v)
if "github.com" in url:
@@ -217,7 +219,6 @@ class TestTarball(unittest.TestCase):
tarball.process_go_archives(go_archives)
self.assertEqual(go_archives, go_archives_expected)
@patch('tarball.config', Mock())
def test_process_multiver_archives(self):
"""Test for tarball.process_multiver_archives method."""
# Set up input values
@@ -236,8 +237,9 @@ class TestTarball(unittest.TestCase):
]
# Set up a return value for parse_config_versions method
attrs = {'parse_config_versions.return_value': config_versions}
tarball.config.configure_mock(**attrs)
tarball.process_multiver_archives(main_src, multiver_archives)
conf = Mock()
conf.configure_mock(**attrs)
tarball.process_multiver_archives(main_src, multiver_archives, conf)
self.assertEqual(multiver_archives, expected_multiver_archives)
@patch('tarball.Source.set_prefix', Mock())