archive repository

Add support for qmake5 build pattern that uses the qmake binary in
/usr/lib64/qt5.

Signed-off-by: William Douglas <william.douglas@intel.com>

.github/actions/clearlinux-latest-action/Dockerfile

@@ -0,0 +1,4 @@
+FROM clearlinux:latest
+RUN swupd bundle-add package-builder python-extras
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/clearlinux-latest-action/action.yml

@@ -0,0 +1,10 @@
+name: 'Clear in Docker'
+description: 'Run commands in the latest Clear Linux OS Docker image'
+inputs:
+  testfunc:
+    description: 'Test function to run'
+    required: true
+    default: 'run_flake8'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/clearlinux-latest-action/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -lx
+
+run_flake8() {
+	make check
+}
+
+run_unittests() {
+	make unittests-no-coverage
+}
+
+if t=$(type -t "$INPUT_TESTFUNC"); then
+	if [ "$t" = "function" ]; then
+		$INPUT_TESTFUNC
+	fi
+fi

.github/workflows/tests.yml

@@ -0,0 +1,27 @@
+name: Autospec Tests
+
+on: [push, pull_request]
+
+jobs:
+  test_style:
+    runs-on: ubuntu-latest
+    name: Flake8
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.3.0
+      - name: Run Tests
+        uses: ./.github/actions/clearlinux-latest-action
+        with:
+          testfunc: run_flake8
+  test_unit:
+    runs-on: ubuntu-latest
+    name: Unit
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.3.0
+      - name: Run Tests
+        uses: ./.github/actions/clearlinux-latest-action
+        with:
+          testfunc: run_unittests

.gitignore

@@ -1,2 +1,7 @@
 __pycache__
 *.pyc
+*~
+*.swp
+tags
+.coverage
+htmlcov

Makefile

@@ -1,2 +1,59 @@
-check: *.py
-	@python3 /usr/bin/flake8 --max-line-length=199 --ignore=E402 $^
+check: autospec/*.py
+	@flake8 --ignore=B902,D100,I201 $^
+
+test_download:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_download.py
+
+test_pkg_integrity:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_pkg_integrity.py
+
+test_tarball:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_tarball.py
+
+test_specfile:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_specfile.py
+
+test_abireport:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_abireport.py
+
+test_commitmessage:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_commitmessage.py
+
+test_files:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_files.py
+
+test_license:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_license.py
+
+test_config:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_config.py
+
+test_build:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_build.py
+
+test_buildreq:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_buildreq.py
+
+test_specdescription:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_specdescription.py
+
+test_count:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_count.py
+
+test_check:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_check.py
+
+test_util:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_util.py
+
+test_general:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_general.py
+
+unittests:
+	PYTHONPATH=${CURDIR}/autospec coverage run -m unittest discover -b -s tests -p 'test_*.py' && coverage report
+
+unittests-no-coverage:
+	PYTHONPATH=${CURDIR}/autospec python3 -m unittest discover -b -s tests -p 'test_*.py'
+
+coverage:
+	coverage report -m

README.md

@@ -0,0 +1,7 @@
+## DISCONTINUATION OF PROJECT. 
+
+This project will no longer be maintained by Intel. 
+
+Intel will not provide or guarantee development of or support for this project, including but not limited to, maintenance, bug fixes, new releases or updates. Patches to this project are no longer accepted by Intel. If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the community, please create your own fork of the project.
+
+Contact: webadmin@linux.intel.com  

README.rst

@@ -2,23 +2,25 @@
 Autospec
 ========
 
-autospec is a tool to assist in the automated creation and maintainence
-of RPM packaging. It will continuously run updated builds based on new
-information discovered from build failures, until it has a complete and
-valid .spec file. The tool makes use of mock to achieve this.
+autospec is a tool to assist in the automated creation and maintenance of RPM
+packaging. It will continuously run updated builds based on new information
+discovered from build failures until it has a complete and valid .spec file. The
+tool makes use of mock to achieve this.
+
+.. contents:: Table of Contents
 
 License
 =======
 autospec is available under the terms of the GPL, version 3.0
 
-Copyright (C) 2015 Intel Corporation
+Copyright (C) 2017 Intel Corporation
 
 
 Configuration of autospec
 =========================
 autospec is configured by means of a simple INI-style configuration file.
-The default location of this file is assumed to be ``common/autospec.conf``,
-relative to the directory in which autospec is executed.
+The default location of this file is assumed to be
+``/usr/share/defaults/autospec/autospec.conf``.
 
 Example ``autospec.conf`` file::
 
@@ -26,40 +28,67 @@ Example ``autospec.conf`` file::
     git = git@someurl.com/%(NAME)s
     license_fetch = http://yourhost/hash.php
     license_show = http://yourhost/showone.php?hash=%(HASH)s
+    packages_file = file:///path/to/package_list_file
+    yum_conf = file:///path/to/yum.conf
     upstream = http://yourhost/tarballs/%(HASH)s/%(NAME)s
 
+git
+  Optional URI template for remote git repository
 
-**git**
-    The upstream git repository URL base
+license_fetch
+  Optional URL to use for scanning license files
 
-**license_fetch**
-    Optional URL to use for scanning license files
+license_show
+  Optional URL to interact with online license checker
 
-**license_show**
-    Optional URL to interact with online license checker
+packages_file
+  Optional path to add autodetected runtime requirement checking
 
-**upstream**
-    Base URL for stored upstream tarballs
+yum_conf
+  Optional path to yum configuration
+
+upstream
+  Base URL for stored upstream tarballs
 
 Synopsis
 ========
 
-Usage: ``python3 autospec.py [options] URL``
+.. code-block:: bash
 
+  usage: autospec.py [-h] [-g] [-n NAME] [-v VERSION]
+                     [-a [ARCHIVES [ARCHIVES ...]]] [-l] [-b] [-c CONFIG]
+                     [-t TARGET] [-i] [-p] [--non_interactive] [-C]
+                     [-m MOCK_CONFIG] <url>
 
--h, --help                                      show help message and exit
--g, --skip-git                                  Don't commit result to git
--n NAME, --name NAME                            Override the package name
--a ARCHIVES, --archives ARCHIVES
-                                                tarball URLs for additional source archives and a
-                                                location for the sources to be extacted to (e.g.
-                                                http://example.com/downloads/dependency.tar.gz
-                                                /directory/relative/to/extract/root )
--l, --license-only                              Only scan for license files
--b, --skip-bump                                 Don't bump release number
--c CONFIG, --config CONFIG                      Set configuration file to use
--t DIRECTORY --target DIRECTORY                 Set location to create or use
+    url                 (required) tarball URL
+                        (e.g. http://example.com/downloads/mytar.tar.gz)
 
+optional arguments:
+  -h, --help            show this help message and exit
+  -g, --skip-git        Don't commit result to git
+  -n NAME, --name NAME  Override the package name
+  -v VERSION, --version VERSION
+                        Override the package version
+  -a ARCHIVES, --archives ARCHIVES
+                        tarball URLs for additional source archives and a
+                        location for the sources to be extacted to (e.g.
+                        http://example.com/downloads/dependency.tar.gz
+                        /directory/relative/to/extract/root )
+  -l, --license-only    Only scan for license files
+  -b, --skip-bump       Don't bump release number
+  -c CONFIG, --config CONFIG
+                        Set configuration file to use
+  -t TARGET, --target TARGET
+                        Target location to create or reuse
+  -i, --integrity       Search for package signature from source URL and
+                        attempt to verify package
+  -p, --prep-only       Only perform preparatory work on package
+  --non_interactive     Disable interactive mode for package verification
+  -C, --cleanup         Clean up mock chroot after building the package
+  -m MOCK_CONFIG, --mock-config MOCK_CONFIG
+                        Value to pass with Mock's -r option. Defaults to
+                        "clear", meaning that Mock will use
+                        /etc/mock/clear.cfg.
 
 
 Requirements
@@ -67,27 +96,26 @@ Requirements
 
 In order to run correctly, ``autospec`` requires the following components:
 
- * python3
- * correctly configured mock
+* python3
+* correctly configured mock
 
-If ``autospec`` is not configured to use a license server, then you will
-need a ``common/licenses`` file -  which should be an up to date list of
-licenses to facilitate automatic license detection during the scan of a
-tarball. For correctness, license names should be in the SPDX identifier
-format. Each line in the file constitutes a license definition, for example::
+If ``autospec`` is not configured to use a license server, then it will use the
+``autospec/license_hashes`` file -  which is a list of licenses to facilitate
+automatic license detection during the scan of a tarball. For correctness,
+license names should be in the SPDX identifier format. Each line in the file
+constitutes a license definition, for example::
 
-    750b9d9cc986bfc80b47c9672c48ca615cac0c87 | BSD-3-Clause
-    175e59be229a5bedc6be93e958a970385bb04a62 | Apache-2.0
-    794a893e510ca5c15c9c97a609ce47b0df74fc1a | BSD-2-Clause
+    750b9d9cc986bfc80b47c9672c48ca615cac0c87, BSD-3-Clause
+    175e59be229a5bedc6be93e958a970385bb04a62, Apache-2.0
+    794a893e510ca5c15c9c97a609ce47b0df74fc1a, BSD-2-Clause
 
 
 Control files
 ==============
-
-It is possible to influence precisely how autospec will behave in order to
-gain fine control over the build itself. These files may be used to alter
-the default behaviour of the configure routine, to blacklist build dependencies
-from being automatically added, and such.
+It is possible to influence precisely how autospec will behave in order to gain
+fine control over the build itself. These files may be used to alter the default
+behaviour of the configure routine, to blacklist build dependencies from being
+automatically added, and such.
 
 These files are expected to live in same directory that the resulting ``.spec``
 will live.
@@ -95,215 +123,511 @@ will live.
 Common files
 ------------
 
-**release**
+release
+  This file contains the current release number that will be used in the
+  ``.spec``. This is also bumped and generated on existing and new packages,
+  respectively. This results in less manual work via automatic management.
 
-    This file contains the current release number that will be used in the
-    ``.spec``. This is also bumped and generated on existing and new packages,
-    respectively. This results in less manual work via automatic management.
+license_skips
+  Each line in the file should be the path to a license file. That path needs
+  to account for the package tarfile prefix. Likely requires using '*' to be
+  effective (e.g. ``pkgname-*/path/to/license`` where ``*`` handles the version).
 
-**$package.license**
+  Files paths can contain a single '*' per directory such that
+  a line of ``/foo*/bar*`` is allowed but ``/foo*bar*`` is not.
 
-    In certain cases, the package license may not be automatically discovered.
-    In this instance, ``autospec`` will exit with an error. Update this file
-    to contain the valid SPDX identifier for any license(s) for the package,
-    replacing ``$package`` in the filename with the actual package name.
+$package.license
+  In certain cases, the package license may not be automatically discovered.  In
+  this instance, ``autospec`` will exit with an error. Update this file to
+  contain the valid SPDX identifier for any license(s) for the package,
+  replacing ``$package`` in the filename with the actual package name.
 
 Controlling dependencies
 -------------------------
 
-**buildreq_add**
+buildreq_add
+  Each line in the file provides the name of a package to add as a build
+  dependency to the ``.spec``.
 
-    Each line in the file provides the name of a package to add
-    as a build dependency to the ``.spec``.
+pkgconfig_add
+  Each line in the file is assumed to be a pkgconfig() build dependency.  Add
+  the pkg-config names here, as ``autospec`` will automatically transform the
+  names into their ``pkgconfig($name)`` style when generating the ``.spec``.
 
-**pkgconfig_add**
+requires_add
+  Each line in the file provides the name of a package to add as a runtime
+  dependency to the ``.spec``.
 
-    Each line in the file is assumed to be a pkgconfig() build dependency.
-    Add the pkg-config names here, as ``autospec`` will automatically transform
-    the names into their ``pkgconfig($name)`` style when generating the ``.spec``.
+${custom}_requires_add
+  Same as "requires_add" above, but instead of the Requires being placed on the
+  ``main`` subpackage, they will be placed on the ``-${custom}`` subpackage.
 
-**buildreq_ban**
+provides_add
+  Each line in the file provides the name of a identifier to add as a Provides
+  to the ``.spec``.
 
-    Each line in the file is a build dependency that under no circumstance
-    should be automatically added to the build dependencies. This is useful
-    to block automatic configuration routines adding undesired functionality,
-    or to omit any automatically discovered dependencies during tarball scanning.
+${custom}_provides_add
+  Same as "provides_add" above, but instead of the Provides being placed on the
+  ``main`` subpackage, they will be placed on the ``-${custom}`` subpackage.
 
-**pkgconfig_ban**
+buildreq_ban
+  Each line in the file is a build dependency that under no circumstance should
+  be automatically added to the build dependencies. This is useful to block
+  automatic configuration routines adding undesired functionality, or to omit
+  any automatically discovered dependencies during tarball scanning.
 
-    Each line in this file is a pkgconfig() build dependency that should not
-    be added automatically to the build, much the same as ``buildreq_ban``.
-    As with ``pkgconfig_add``, these names are automatically transformed by
-    ``autospec`` into their correct ``pkgconfig($name))`` style.
+pkgconfig_ban
+  Each line in this file is a pkgconfig() build dependency that should not be
+  added automatically to the build, much the same as ``buildreq_ban``.  As with
+  ``pkgconfig_add``, these names are automatically transformed by ``autospec``
+  into their correct ``pkgconfig($name))`` style.
 
+requires_ban
+  Each line in the file is a runtime dependency that under no circumstance
+  should be automatically added to the runtime dependencies. This is useful to
+  block automatic configuration routines adding undesired functionality, or to
+  omit any automatically discovered dependencies during tarball scanning.
+
+${custom}_requires_ban
+  Same as "requires_ban" above, but instead of the Requires being removed from
+  the ``main`` subpackage, they will be removed from the ``-${custom}``
+  subpackage.
+
+provides_ban
+  Each line in the file is an identifier that under no circumstance should be
+  automatically added as a Provides. This is useful to block automatic
+  configuration routines adding undesired identifiers.
+
+${custom}_provides_ban
+  Same as "provides_ban" above, but instead of the Provides being removed from
+  the ``main`` subpackage, they will be removed from the ``-${custom}``
+  subpackage.
+
+.. note::
+
+  Run time requirements are not assumed to be build time requirement
+  If a package has the same build and run time requirement it must be added
+  to both buildreq_add and requires_add.
 
 Controlling the build process
 ------------------------------
 
-**configure**
+invalid_release_sig
+  This file contains the current version that will **not** have its package
+  file be processed for signature verification (overriding the config_opt).
+  This file will be automatically deleted after a new release and is intended
+  to override a single bad signed release.
 
-    This file contains configuration flags to pass to the ``%configure``
-    macro for autotools based tarballs. As an example, adding ``--disable-static``
-    to ``./configure`` for an autootools based tarball would result in
-    ``%configure --disable-static`` being emitted in the ``.spec``.
+extra_sources
+  This file contains a list of extra files to be added to the ``.spec`` and
+  optionally installed as well. Each non-blank and non-comment line should start
+  with the file name as found in the Git directory, followed by arguments to be
+  passed to the /usr/bin/install(3) command, with at least one argument starting
+  with a slash, denoting the destination directory (there's no need for
+  ``%{buildroot}``). If the install arguments are missing, Autospec will not
+  generate an installation command and the package should specify how to install
+  in the install_append file (see below).
 
-**cmake_args**
+configure
+  This file contains configuration flags to pass to the ``%configure`` macro for
+  autotools based tarballs. As an example, adding ``--disable-static`` to
+  ``./configure`` for an autootools based tarball would result in ``%configure
+  --disable-static`` being emitted in the ``.spec``.
 
-    This file contains arguments that should be passed to the ``%cmake``
-    macro for CMake based tarballs. As an example, adding ``-DUSE_LIB64=ON`` to
-    ``./cmake_args`` would result in ``%cmake -DUSE_LIB64=ON`` being emitted
-    in the ``.spec``.
+configure_openmpi
+  This file contains configuration flags to pass to the ``%configure`` macro for
+  autotools based tarballs to configure openmpi builds.
 
-**broken_parallel_build**
+configure32, configure64, configure_avx2, configure_avx512
+  These files are appended to the ``%configure'' macro after the
+  contents of the ``configure'' file above. They are used for 32-bit,
+  regular 64-bit, AVX2 and AVX512 builds, respectively.
 
-    If this file exists, then parallelisation will be disabled in the build.
-    This usually means that ``%{?_smp_mflags}`` will not be passed to ``make``
+cmake_args
+  This file contains arguments that should be passed to the ``%cmake`` macro for
+  CMake based tarballs. As an example, adding ``-DUSE_LIB64=ON`` to
+  ``./cmake_args`` would result in ``%cmake -DUSE_LIB64=ON`` being emitted in
+  the ``.spec``.
 
-**make_args**
+cmake_args_openmpi
+  This file contains arguments that should be passed to the ``%cmake`` macro for
+  CMake based tarballs for openmpi builds.
 
-    The contents of this file are appended to the ``make`` invocation. This
-    may be useful for passing arguments to ``make``, i.e. ``make TOOLDIR=/usr``
+make_command
+  The contents of this file will be used instead of the ``make``
+  command, i.e. use this if ``make`` should be replace with another build tool
+  like ``ninja``.
 
-**make_install_args**
+make_args
+  The contents of this file are appended to the ``make`` invocation. This may be
+  useful for passing arguments to ``make``, i.e. ``make TOOLDIR=/usr``
 
-    Much like ``make_args``, this will pass arguments to the ``make install``
-    macro in the ``.spec``
+make32_args
+  The contents of this file are appended to the ``make`` invocation of the 32bit
+  build. It is appended after the make_args content so 32bit specific overrides
+  can be added.
 
-**install_macro**
+make_install_args
+  Much like ``make_args``, this will pass arguments to the ``make install``
+  macro in the ``.spec``
 
-    The contents of this file be used instead of the automatically detected
-    ``install`` routine, i.e. use this if ``%make_install`` is insufficient.
+make32_install_args
+  Much like ``make32_args``, this will pass arguments to the ``make install``
+  macro in the ``.spec`` for the 32bit build. Again it is appended after
+  make_install_args so 32bit specific overrides can be added.
 
-**subdir**
+prep_prepend
+  Additional actions that should take place directly after ``%prep``
+  and before the ``%setup`` macro.  This will be placed in the
+  resulting ``.spec``, and is used for situations where fine-grained
+  control is required.
 
-    Not all packages have their ``Makefile``'s available in the root of the tarball.
-    An example of this may be cross-platform projects that split Makefile's into
-    the ``unix`` subdirectory. Set the name in this file and the ``.spec`` will
-    emit the correct ``pushd`` and ``popd`` lines to utilise these directories
-    for each step in the build.
+copy_prepend
+  Additional actions that should take place directly before the source
+  directory is copied for other builds (32bit, avx2, etc). This will be
+  placed in the resulting ``.spec``, and is used for situations where
+  fine-grained control is required.
 
-**build_pattern**
+build_prepend
+  Additional actions that should take place after ``%build`` and before
+  the ``%configure`` macro or equivalent (``%cmake``, etc.). If autospec
+  is creating AVX2, AVX-512 or 32-bit, these actions will be repeated for
+  each of those builds, This will be placed in the resulting ``.spec``,
+  and is used for situations where fine-grained control is required.
 
-    In certain situations, the automatically detected build pattern may not
-    work for the given package. This one line file allows you to override the
-    build pattern that ``autospec`` will use. The supported build_pattern types are:
+build_prepend_once
+  Additional actions that should take place directly after ``%build``
+  and before the ``%configure`` macro or equivalent (``%cmake``, etc.).
+  If autospec is creating AVX2, AVX-512 or 32-bit, these action will
+  not be repeated for each of those builds, This will be placed in the
+  resulting ``.spec``, and is used for situations where fine-grained
+  control is required.
 
-        - configure: Traditional ``%configure`` autotools route
-        - configure_ac: Like ``configure, but performs ``%reconfigure`` to regenerate ``./configure``
-        - autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh`` instead of ``%reconfigure``
-        - distutils: Only build the Pythonic package with Python 2
-        - distutils23: Build the Pythonic package using both Python 2 and Python 3
+make_prepend
+  Additional actions that should take place directly after the
+  configuring step and before the ``%make`` macro or equivalent. If
+  autospec is creating AVX2, AVX-512 or 32-bit, these actions will be
+  repeated for each of those builds, before their respective make
+  steps. This will be placed in the resulting ``.spec``, and is used
+  for situations where fine-grained control is required.
 
-**series**
+install_prepend
+  Additional actions that should take place directly after
+  ``%install`` but before the ``%make_install`` macro (or equivalent).
+  This will be placed in the resulting ``.spec``, and is used for
+  situations where fine-grained control is required.
 
-    This file contains a list of patches to apply during the build, using the ``%patch``
-    macro. As such it is affected by ``-p1`` style modifiers.
+install_append
+  Additional actions that should take place at the very end of the
+  ``%install`` section. This will be placed in the resulting ``.spec``,
+  and is used for situations where fine-grained control is required.
+
+install_macro
+  The contents of this file will be used instead of the automatically detected
+  ``install`` routine, i.e. use this if ``%make_install`` is insufficient.
+
+subdir
+  Not all packages have their ``Makefile``'s available in the root of the
+  tarball.  An example of this may be cross-platform projects that split
+  Makefile's into the ``unix`` subdirectory. Set the name in this file and the
+  ``.spec`` will emit the correct ``pushd`` and ``popd`` lines to utilise these
+  directories for each step in the build.
+
+cmake_srcdir
+ The contents of this file are a path to the source directory in which to run
+ cmake for non-standard packages. This path is relative to the clr-build
+ subdirectory, which is created directly below the source package's root.
+
+build_pattern
+  In certain situations, the automatically detected build pattern may not work
+  for the given package. This one line file allows you to override the build
+  pattern that ``autospec`` will use. The supported build_pattern types are:
+
+  * R: R language package
+  * cpan: perl language package
+  * configure: Traditional ``%configure`` autotools route
+  * configure_ac: Like ``configure``, but performs ``%reconfigure`` to
+    regenerate ``./configure``
+  * autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh``
+    instead of ``%reconfigure``
+  * cmake: Traditional builds using CMake
+  * qmake: qmake (Qt5) projects
+  * make: Run ``make`` followed by ``make install``, skipping configure. Note
+    that this is the fallback build pattern in case no other build patterns are
+    autodetected
+  * distutils3: Only build the Pythonic package with Python 3
+  * cargo: Run ```cargo``` to build and install content put in ~/.cargo/bin
+  * pyproject: Build the Pythonic package using the PEP 516 method
+  * meson: Build package with Meson/Ninja
+  * \[WIP\] scons: Build package with Scons
+
+series
+  This file contains a list of patches to apply during the build, using the
+  ``%patch`` macro. As such it is affected by ``-p1`` style modifiers.
+  Arguments to patch can be added after the patch filename.  For example:
+
+  ```
+  0001-my-awesome-patch.patch -d some/subdir -p1
+  ```
+
+pypi_overrides
+  This file contains a list of modules to remove version tests on during the
+  build. It also modifies the requirement of the requires.txt egg if it
+  exists in the package. For example:
+
+  ```
+  colorama
+  ```
+  in the file will cause this modification:
+  ```
+  -    'colorama>=0.2.5,<0.4.4',
+  +    'colorama',
+  ```
+
+service_restart
+  Each line in the file specifies the full path to a systemd unit file
+  installed by this package that should be restarted by clr-service-restart_.
+
+.. _clr-service-restart: https://github.com/clearlinux/clr-service-restart
 
 Controlling files and subpackages
 ---------------------------------
 
-**excludes**
+excludes
+  This file is used to generate ``%exclude`` lines in the ``.spec``. This is
+  useful for omitting files from being included in the resulting package.  Each
+  line in the file should be a full path name.
 
-    This file is used to generate ``%exclude`` lines in the ``.spec``. This
-    is useful for omitting files from being included in the resulting package.
-    Each line in the file should be a full path name.
+extras
+  Each line in the file should be a full path within the resulting package, that
+  you wish to be placed into an automatic ``-extras`` subpackage. This allows
+  one to keep the main package slim and split out optional functionality or
+  files.
 
-**keepstatic**
+  Files paths can contain a single '*' per directory such that
+  a line of ``/foo*/bar*`` is allowed but ``/foo*bar*`` is not.
 
-    If this file exists, then ``%define keepstatic 1`` is emitted in the ``.spec``.
-    As a result, any static archive (``.a``) files will not be removed by rpmbuild.
+dev_extras
+  Same as "extras" above, but instead of the files being placed in an
+  ``-extras`` subpackage, they will be placed in the ``-dev`` one. Use this
+  functionality to place files used only for development against this
+  software that Autospec does not automatically detect.
 
-**extras**
+tests_extras
+  Same as "extras" above, but instead of the files being placed in an
+  ``-extras`` subpackage, they will be placed in the ``-tests`` one. Use this
+  functionality to place files used only for testing against this
+  software that Autospec does not automatically detect.
 
-    Each line in the file should be a full path within the resulting package,
-    that you wish to be placed into an automatic ``-extras`` subpackage. This
-    allows one to keep the main package slim and split out optional functionality
-    or files.
+${custom}_extras
+  Same as "extras" above, but instead of the files being placed in an
+  ``-extras`` subpackage, they will be placed in the ``extras-${custom}``
+  subpackage.
 
-**setuid**
+${custom}_extras_requires
+  Each line contains a subpackage names of other subpackages in the package.
+  This is used when the ``extras-${custom}`` subpackage has a runtime
+  requirement on a sibling subpackage.
 
-    Each line in this file should contain the full path to a binary in the resulting
-    build that should have the ``setuid`` attribute set with the ``%attr`` macro.
+  An example of the ``${custom}_extras`` and ``${custom}_extras_requires``
+  being used together with::
 
-**attrs**
+    /usr/bin/foo
 
-    Each line in this file should be a full ``%attr`` macro line that will be included
-    in the ``.spec`` to have fine-grained control over the permissions and ownership
-    of files in the package.
+  in foo_extras and::
+
+    data
+
+  in foo_extras_requires will produce a spec file package
+  section for example-foo-extras with the following content::
+
+    %package extras-foo
+    Summary: extras-foo components for the example package.
+    Group: Default
+    Requires: example-data = %{version}-%{release}
+
+    %description extras-foo
+    extras-foo components for the example package.
+
+setuid
+  Each line in this file should contain the full path to a binary in the
+  resulting build that should have the ``setuid`` attribute set with the
+  ``%attr`` macro.
+
+attrs
+  Each line in this file should specify mode, user, group and filename
+  (space separated) which is translated into a full ``%attr`` macro
+  line that will be included in the ``.spec`` to have fine-grained control
+  over the permissions and ownership of files in the package.
+
+  An example of a ``attrs`` file would contain::
+
+    4755 root messagebus /usr/libexec/dbus-daemon-launch-helper
+
+  which would translate to the following line in the resulting ``.spec`` file::
+
+    %attr(4755,root,messagebus) /usr/libexec/dbus-daemon-launch-helper
 
 
 Controlling test suites
 -----------------------
-
 By default, ``autospec`` will attempt to detect potential test suites that
 can be run in the ``%check`` portion of the ``.spec``.
 
-**skip_test_suite**
+make_check_command
+  Override or set the command to use in the ``%check`` portion of the ``.spec``.
+  This may be useful when a package uses a custom test suite, or requires
+  additional work/parameters, to work correctly.
 
-    If this file exists, ``autospec`` will not emit any ``%check`` functionality.
+Controlling miscellaneous spec metadata
+---------------------------------------
 
-**unit_tests_must_pass**
+description
+  Provides content for the %description section, overriding the content
+  autospec autodetects. This is useful if autospec cannot find proper content
+  for the description, if one wants to customize the content for better
+  presentation, etc.
 
-    This file is automatically created upon successful completion of a package build.
-    This allows one to identify regressions in test failures when updating or
-    altering a package.
-    ``autospec`` will fail a package that does not pass it's test suite if this file
-    exists.
+summary
 
-**make_check_command**
+  Provides the main Summary: value of the package, overriding any automatically
+  found values. Only the first line is used.
 
-    Override or set the command to use in the ``%check`` portion of the ``.spec``.
-    This may be useful when a package uses a custom test suite, or requires
-    additional work/parameters, to work correctly.
-
-**allow_test_failures**
-
-    The existence of this file will allow test failures, and will still emit
-    the ``%check`` code in a way that allows the build to continue.
+pypi.json
+  Provides an alternative to reading the pypi api url for package metadata.
+  provides, requires, summary, description and license information could be
+  sourced from this file (see https://wiki.python.org/moin/PyPIJSON) for more
+  details on the structure.
 
 
-Controlling flags and optimisation
------------------------------------
+Controlling flags and optimization
+----------------------------------
+Further control of the build can be achieved through the use of the
+``options.conf`` file. If this file does not exist it is created by autospec
+with default values. If certain deprecated configuration files exists autospec
+will use the value indicated by those files and remove them.
 
-**asneeded**
+The options that can be set in ``options.conf`` are as follows:
 
-    If this file exists, the ``.spec`` will disable the LD_AS_NEEDED variable.
-    Supporting binutils (such as found in Clear Linux Project for Intel Architecture)
-    will then revert to their normal behaviour, instead of enforcing ``-Wl,-as-needed``
-    in the most correct sense.
+asneeded
+  If this is option set, the ``.spec`` will disable the LD_AS_NEEDED variable.
+  Supporting binutils (such as found in Clear Linux Project for Intel
+  Architecture) will then revert to their normal behaviour, instead of enforcing
+  ``-Wl,-as-needed`` in the most correct sense.
 
-**optimize_size**
+optimize_size
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build the
+  package optimized for *size*, and not for *speed*. Use this when size is more
+  critical than performance.
 
-    If this file exists, the ``CFLAGS/LDFLAGS`` will be extended to build
-    the package optimised for *size*, and not for *speed*. Use this when
-    size is more critical than performance.
+funroll-loops
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build the
+  package optimized for *speed*. In short this where speed is of paramount
+  importance, and will use ``-03`` by default.
 
-**funroll-loops**
+insecure_build
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be **replaced**, using the
+  smallest ``-02`` based generic flags possible. This is useful for operating
+  systems employing heavy optimizations or full RELRO by default.
 
-    If this file exists, the ``CFLAGS/LDFLAGS`` will be extended to build
-    the package optimised for *speed*. In short this where speed is of
-    paramount importance, and will use ``-03`` by default.
+pgo
+  If this option is set, the ``CFLAGS/CXXFLAGS`` will be extended to build the
+  package with profile-guided optimization data. It will add ``-O3``,
+  ``-fprofile-use``, ``-fprofile-correction`` and ``-fprofile-dir=pgo``.
 
-**insecure_build**
+use_lto
+  If this option is set, link time optimization is enabled for the build.
 
-    If this file exists, the ``CFLAGS/LDFLAGS`` will be **replaced**, using
-    the smallest ``-02`` based generic flags possible. This is useful for
-    operating systems employing heavy optimisations or full RELRO by default.
+use_avx2
+  If this option is set, a second set of libraries, for AVX2, is built.
 
+use_avx512
+  If this option is set, an additional set of libraries, for AVX512, is built.
+
+openmpi
+  If this option is set, an additional openmpi package is built.
+
+fast-math
+  If this option is set, -ffast-math is passed to the compiler.
+
+broken_c++
+  If this option is set, flags are extended with -std=gnu++98.
+
+allow_test_failures
+  If this option is set it will allow test failures, and will still emit the
+  ``%check`` code in a way that allows the build to continue.
+
+skip_tests
+  If this option is set the test suite will not be run.
+
+no_autostart
+  If this option is set the autostart subpackage (which contains all files
+  matching /usr/lib/systemd/system/\*.target.wants/) will not be required by the
+  base package.
+
+conservative_flags
+  If this option is set autospec will set conservative build flags
+
+broken_parallel_build
+  If this option is set, the parallelization is disabled during build.
+
+use_clang
+  If this option is set autospec will utilize clang. This unsets the
+  funroll-loops optimization if it is set.
+
+keepstatic
+  If this option is set, then ``%define keepstatic 1`` is emitted in the
+  ``.spec``.  As a result, any static archive (``.a``) files will not be removed
+  by rpmbuild.
+
+32bit
+  This option will trigger the creation of 32-bit libraries for a 32-bit build.
+
+nostrip
+  This option will suppress the stripping of the created binaries.
+
+verify_required
+  This option will make package verification required for the build. This option
+  is automatically set if package verification is ever successful, but can be
+  turned off manually.
+
+security_sensitive
+  This options sets flags for security-sensitive builds.
+
+so_to_lib
+  This option causes package ``.so`` files to be added to the ``lib`` subpackage
+  instead of the ``dev`` subpackage.
+
+dev_requires_extras
+  If this option is set, the ``extras`` subpackage is marked as a dependency of
+  the ``dev`` package.
+
+autoupdate
+  This option indicates that the package is trusted enough to be automatically
+  update to its newest available version when set to ``true``. This flag is
+  intended to be used by tools running autospec automatically.
+
+compat
+  This option indicates the package is a library compatibility package and only
+  provides versioned library files.
+
+nodebug
+  If this option is set, ``debuginfo`` is not created for this package.
 
 Name and version resolution
 ===========================
 
-``autospec`` will attempt to use a number of patterns to determine the name
-and version of the package by examining the URL. For most tarballs this is
-simple, if they are of the format ``$name-$version.tar.$compression``.
+``autospec`` will attempt to use a number of patterns to determine the name and
+version of the package by examining the URL. For most tarballs this is simple,
+if they are of the format ``$name-$version.tar.$compression``.
 
-For websites such as ``bitbucket`` or ``GitHub``, using ``get$`` and ``v$.tar.*``
-style links, the project name itself is used from the URL and the version is
-determined by stripping down the tag.
+For websites such as ``bitbucket`` or ``GitHub``, using ``get$`` and
+``v$.tar.*`` style links, the project name itself is used from the URL and the
+version is determined by stripping down the tag.
 
-CPAN Perl packages, R packages, and rubygems.org rubygems are automatically
-prefixed with their language name: ``perl-``, ``R-`` and ``rubygem-`` respectively.
+CPAN Perl packages, pypi ecosystem packages and R packages are automatically
+prefixed with their respective names: ``perl-``, ``pypi-`` and ``R-``
+respectively.
 
 When these automated detections are not desirable, it is possible to override
 these with the ``--name`` flag when invoking ``autospec``
@@ -318,14 +642,14 @@ talks HTTP.
 
 This URL should accept ``POST`` requests with the following keys:
 
-**hash**
-    Contains the SHA-1 hash of the potential license file being checked.
+hash
+  Contains the SHA-1 hash of the potential license file being checked.
 
-**package**
-    The name of the package being examined
+package
+  The name of the package being examined
 
-**text**
-    The contents of the potential license file
+text
+  The contents of the potential license file
 
 Implementations return a *plain text* response with the SPDX identifier
 of the license, if known. An empty response is assumed to mean that this
@@ -333,4 +657,13 @@ license is unknown, in which case ``autospec`` will emit the ``license_show``
 URL. The implementation should show the now-stored license file via a
 web page, and enable a human to make a decision on the license. This is
 then stored internally, allowing future requests to automatically know
-the license type when this hash is encounted again. 
+the license type when this hash is encountered again.
+
+
+Integration of systemd unit files
+=================================
+``autospec`` can add most systemd template file types by having a file in the
+filename.extension in the build directory. Supported extensions are:
+``mount, service, socket, target, timer, path and tmpfiles``. The files will
+be added as Source# entries and be installed to their appropriate system
+location.

autospec/__init__.py

@@ -1,6 +1,5 @@
-import sys
-import os
+"""Autospec, an automated specfile generation utility."""
 
-__all__ = ["buildreq", "build", "buildpattern", "config", "docs","files",
-           "git","lang", "license", "patches", "specdescription", "tarball",
-           "util", "commitmessage", "test", "patches"]
+__all__ = ["abireport", "buildreq", "build", "config", "files",
+           "git", "lang", "license", "patches", "specdescription",
+           "tarball", "util", "commitmessage", "test", "patches"]

autospec/abireport.py

@@ -0,0 +1,338 @@
+#!/bin/true
+#
+# abireport.py - part of autospec
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Generate a symbols file from all shared libraries encountered, enabling
+# a consistent ABI report for every build. We ensure that everything is
+# appropriately sorted, in that a diff only occurs when the shared libraries
+# in the package themselves actually change too.
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+import util
+
+valid_dirs = ["/usr/lib", "/usr/lib64"]
+
+# For determining .so's
+reg = re.compile(r".* ELF (64|32)\-bit LSB shared object,")
+
+# All dynamic binaries
+valid_dyn = re.compile(r".* ELF (64|32)\-bit LSB (shared object|executable),")
+
+# shared-lib matcher
+shared_lib = re.compile(r".*Shared library: \[(.*)\].*")
+
+wanted_symbol_types = ["A", "T"]
+
+ignored_symbols = [
+    "__bss_start",
+    "_edata",
+    "_end",
+    "_fini",
+    "_init",
+]
+
+
+def get_output(cmd):
+    """Return output from subprocess.getoutput."""
+    try:
+        o = subprocess.getoutput(cmd)
+        return o
+    except Exception as e:
+        print("Error: %s" % e)
+
+
+def get_soname(path):
+    """Use objdump to find the SONAME of a file."""
+    cmd = "objdump -p \"{}\"|grep SONAME".format(path)
+    try:
+        line = get_output(cmd)
+        if "SONAME" not in line:
+            return None
+        line = line.strip()
+        spl = line.split()[1]
+        return spl
+    except Exception:
+        return None
+
+
+def get_shared_dependencies(path):
+    """Return the shared dependencies for a given path."""
+    ret = set()
+    cmd = "readelf -d {}".format(path)
+
+    for line in get_output(cmd).split("\n"):
+        line = line.strip()
+        shared = shared_lib.match(line)
+        if shared is None:
+            continue
+        ret.add(shared.group(1))
+
+    return ret
+
+
+def get_all_dependencies(path):
+    """Determine all dependencies in the given path."""
+    deps = set()
+
+    sonames = set()
+    examine = set()
+
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            fpath = os.path.join(root, file)
+            if not is_dynamic_binary(fpath):
+                continue
+            # Encountered a valid dynamic linked object
+            if is_file_valid(fpath):
+                # We must account for *all* internal symbols due to rpaths and
+                # overriding of LD_LIBRARY_PATH
+                soname = get_soname(fpath)
+                if soname is not None:
+                    sonames.add(soname)
+            if is_dynamic_binary(fpath):
+                examine.add(fpath)
+
+    for path in examine:
+        current_deps = get_shared_dependencies(path)
+        # Ensure we don't add a dependency on an internally provided symbol
+        deps.update(set(filter(lambda s: s not in sonames, current_deps)))
+
+    return deps
+
+
+def get_file_magic(path):
+    """Return the 'magic' for a given path."""
+    cmd = "file \"{}\"".format(path)
+    try:
+        line = get_output(cmd).split("\n")[0]
+        return line.strip()
+    except Exception:
+        return None
+
+
+def is_dynamic_binary(path):
+    """Determine if a given path is a dynamic binary."""
+    if not os.path.exists(path) or not os.path.isfile(path):
+        return False
+    mg = get_file_magic(path)
+    if not mg:
+        return False
+    if valid_dyn.match(mg):
+        return True
+    return False
+
+
+def is_file_valid(path):
+    """Validate file is an SO."""
+    if not os.path.exists(path) or os.path.islink(path):
+        return False
+    mg = get_file_magic(path)
+    if not mg:
+        return False
+    if reg.match(mg):
+        return True
+    return False
+
+
+def dump_symbols(path):
+    """Get symbols from a file."""
+    cmd = "nm --defined-only -g --dynamic \"{}\" | c++filt".format(path)
+    lines = None
+
+    ret = set()
+
+    try:
+        lines = get_output(cmd)
+    except Exception as e:
+        util.print_fatal("Fatal error inspecting {}: {}".format(path, e))
+        sys.exit(1)
+    for line in lines.split("\n"):
+        line = line.strip()
+
+        spl = line.split()
+        if len(spl) != 3:
+            continue
+        sym_type = spl[1]
+        sym_id = spl[2]
+
+        if sym_type not in wanted_symbol_types:
+            continue
+        if sym_id in ignored_symbols:
+            continue
+        ret.add(sym_id)
+    return ret
+
+
+def purge_tree(tree):
+    """Run rm -fr."""
+    if not os.path.exists(tree):
+        return
+    try:
+        shutil.rmtree(tree)
+    except Exception as e:
+        util.print_fatal("Cannot remove tree: {}".format(e))
+        sys.exit(1)
+
+
+def truncate_file(path):
+    """Zero file content."""
+    if not os.path.exists(path):
+        return
+    with util.open_auto(path, "r+") as trunc:
+        trunc.truncate()
+
+
+def examine_abi(download_path, name):
+    """Proxy the ABI reporting to the right function."""
+    download_path = os.path.abspath(download_path)
+    results_dir = os.path.abspath(os.path.join(download_path, "results"))
+
+    if not os.path.exists(results_dir):
+        util.print_fatal("Results directory does not exist, aborting")
+        sys.exit(1)
+
+    if util.binary_in_path("abireport"):
+        examine_abi_host(download_path, results_dir, name)
+    else:
+        util.print_warning("abireport is not installed. Using slow scanning")
+        examine_abi_fallback(download_path, results_dir, name)
+
+
+def examine_abi_host(download_path, results_dir, name):
+    """Make use of the hostside abireport tool."""
+    rpms = set()
+    for item in os.listdir(results_dir):
+        namelen = len(name)
+        if item.find("-extras-", namelen) >= namelen:
+            continue
+        if item.endswith(".rpm") and not item.endswith(".src.rpm"):
+            rpms.add("{}/{}".format(results_dir, item))
+
+    if len(rpms) == 0:
+        util.print_fatal("No usable rpms found, aborting")
+        sys.exit(1)
+
+    try:
+        util.call("abireport scan-packages {}".format(" ".join(rpms)),
+                  cwd=download_path)
+    except Exception as e:
+        util.print_fatal("Error invoking abireport: {}".format(e))
+        sys.exit(1)
+
+
+def examine_abi_fallback(download_path, results_dir, name):
+    """Missing abireport so fallback to internal scanning."""
+    old_dir = os.getcwd()
+
+    rpms = set()
+    for item in os.listdir(results_dir):
+        namelen = len(name)
+        if item.find("-extras-", namelen) >= namelen:
+            continue
+        if item.endswith(".rpm") and not item.endswith(".src.rpm"):
+            rpms.add(os.path.basename(item))
+
+    if len(rpms) == 0:
+        util.print_fatal("No usable rpms found, aborting")
+        sys.exit(1)
+
+    extract_dir = os.path.abspath(os.path.join(download_path, "__extraction"))
+    purge_tree(extract_dir)
+
+    try:
+        os.makedirs(extract_dir)
+    except Exception as e:
+        util.print_fatal("Cannot create extraction tree: {}".format(e))
+        sys.exit(1)
+
+    os.chdir(extract_dir)
+
+    # Extract all those rpms to our current directory
+    try:
+        for rpm in rpms:
+            cmd = "rpm2cpio \"{}\" | cpio -imd 2>/dev/null".format(os.path.join(results_dir, rpm))
+            subprocess.check_call(cmd, shell=True)
+    except Exception as e:
+        util.print_fatal("Error extracting RPMS: {}".format(e))
+        sys.exit(1)
+
+    os.chdir(download_path)
+    collected_files = set()
+
+    # Places we expect to find shared libraries
+    for check_path in valid_dirs:
+        if check_path[0] == '/':
+            check_path = check_path[1:]
+
+        dirn = os.path.join(extract_dir, check_path)
+        if not os.path.isdir(dirn):
+            continue
+
+        for file in os.listdir(dirn):
+            f = os.path.basename(file)
+
+            clean_path = os.path.abspath(os.path.join(dirn, f))
+            if not is_file_valid(clean_path):
+                continue
+            collected_files.add(clean_path)
+
+    abi_report = dict()
+
+    # Now examine these libraries
+    for library in sorted(collected_files):
+        soname = get_soname(library)
+        if not soname:
+            warn = "Failed to determine soname of: {}".format(library)
+            util.print_warning(warn)
+            soname = os.path.basename(library)
+        symbols = dump_symbols(library)
+        if symbols and len(symbols) > 0:
+            if soname not in abi_report:
+                abi_report[soname] = set()
+            abi_report[soname].update(symbols)
+
+    report_file = os.path.join(download_path, "symbols")
+
+    if len(abi_report) > 0:
+        # Finally, write the report
+        report = util.open_auto(report_file, "w")
+        for soname in sorted(abi_report.keys()):
+            for symbol in sorted(abi_report[soname]):
+                report.write("{}:{}\n".format(soname, symbol))
+
+        report.close()
+    else:
+        truncate_file(report_file)
+
+    # Write the library report
+    lib_deps = get_all_dependencies(extract_dir)
+    report_file = os.path.join(download_path, "used_libs")
+    if len(lib_deps) > 0:
+        report = util.open_auto(report_file, "w")
+        for soname in sorted(lib_deps):
+            report.write("{}\n".format(soname))
+        report.close()
+    else:
+        truncate_file(report_file)
+
+    os.chdir(old_dir)
+    purge_tree(extract_dir)

autospec/autospec.py

@@ -17,115 +17,113 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
-import sys
+import configparser
 import os
 import re
-import types
+import sys
+import tempfile
 
 import build
-import buildpattern
 import buildreq
+import check
+import commitmessage
 import config
 import files
 import git
-import lang
 import license
-import docs
-import patches
+import pkg_integrity
+import pkg_scan
 import specdescription
+import specfiles
 import tarball
-import test
-import commitmessage
-
-from tarball import name
-from util import _file_write
+from abireport import examine_abi
+from logcheck import logcheck
+from util import binary_in_path, print_build_failed, print_fatal, write_out
 
 sys.path.append(os.path.dirname(__file__))
 
-def write_sources(file):
-    """Append additonal source files.
-    systemd unit files, gcov and additional source tarballs
-    are the currently supported additonal source file types."""
-    source_count = 1
-    for source in sorted(buildpattern.sources["unit"] +
-                         buildpattern.sources["archive"] +
-                         buildpattern.sources["tmpfile"] +
-                         buildpattern.sources["gcov"]):
-        buildpattern.source_index[source] = source_count
-        file.write("Source{0}  : {1}\n".format(source_count, source))
-        source_count += 1
+
+def check_requirements(use_git):
+    """Ensure all requirements are satisfied before continuing."""
+    required_bins = ["mock", "rpm2cpio", "nm", "objdump", "cpio", "readelf"]
+
+    if use_git:
+        required_bins.append("git")
+
+    missing = [x for x in required_bins if not binary_in_path(x)]
+
+    if missing:
+        print_fatal("Required programs are not installed: {}".format(", ".join(missing)))
+        sys.exit(1)
 
 
-def write_spec(filename):
-    file = open(filename, "w", encoding="utf-8")
-    file.write_strip = types.MethodType(_file_write, file)
-    file.write("#\n")
-    file.write("# This file is auto-generated. DO NOT EDIT\n")
-    file.write("# Generated by: autospec.py\n")
-    file.write("#\n")
-
-    if config.keepstatic == 1:
-        file.write("%define keepstatic 1\n")
-
-    # first, write the general package header
-    tarball.write_nvr(file)
-    write_sources(file)
-    specdescription.write_summary(file)
-    license.write_license(file)
-
-    files.write_main_subpackage_requires(file)
-    buildreq.write_buildreq(file)
-    patches.write_patch_header(file)
-
-    # then write the main package extra content
-    specdescription.write_description(file)
-    files.write_files_header(file)
-
-    # then write the build instructions
-    buildpattern.write_buildpattern(file)
-
-    # then write the scriplets
-    files.write_scriplets(file)
-
-    # then write the %files
-    files.write_files(file)
-    lang.write_lang_files(file)
-
-    file.close()
+def load_specfile(conf, specfile):
+    """Gather all information from static analysis into Specfile instance."""
+    specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ)
+    license.load_specfile(specfile)
+    check.load_specfile(specfile)
 
 
-def add_sources(download_path, archives):
-    for file in os.listdir(download_path):
-        if re.search(".*\.(mount|service|socket|target)$", file):
-            buildpattern.sources["unit"].append(file)
-    buildpattern.sources["unit"].sort()
-    #
-    # systemd-tmpfiles uses the configuration files from
-    # /usr/lib/tmpfiles.d/ directories to describe the creation,
-    # cleaning and removal of volatile and temporary files and
-    # directories which usually reside in directories such as
-    # /run or /tmp.
-    #
-    if os.path.exists(os.path.normpath(build.download_path +
-                                       "/{0}.tmpfiles".format(tarball.name))):
-        buildpattern.sources["tmpfile"].append(
-            "{}.tmpfiles".format(tarball.name))
-    if tarball.gcov_file:
-        buildpattern.sources["gcov"].append(tarball.gcov_file)
-    for archive, destination in zip(archives[::2], archives[1::2]):
-        buildpattern.sources["archive"].append(archive)
-        buildpattern.archive_details[archive + "destination"] = destination
+def read_old_metadata():
+    """Handle options.conf providing package, url and archives."""
+    if not os.path.exists(os.path.join(os.getcwd(), 'options.conf')):
+        return None, None, []
+
+    config_f = configparser.ConfigParser(interpolation=None)
+    config_f.read('options.conf')
+    if "package" not in config_f.sections():
+        return None, None, []
+
+    archives = config_f["package"].get("archives")
+    archives = archives.split() if archives else []
+    print("ARCHIVES {}".format(archives))
+
+    return (config_f["package"].get("name"),
+            config_f["package"].get("url"),
+            archives)
+
+
+def save_mock_logs(path, iteration):
+    """Save Mock build logs to <path>/results/round<iteration>-*.log."""
+    basedir = os.path.join(path, "results")
+    loglist = ["build", "root", "srpm-build", "srpm-root", "mock_srpm", "mock_build"]
+    for log in loglist:
+        src = "{}/{}.log".format(basedir, log)
+        dest = "{}/round{}-{}.log".format(basedir, iteration, log)
+        os.rename(src, dest)
+
+
+def write_prep(conf, workingdir, content):
+    """Write metadata to the local workingdir when --prep-only is used."""
+    if conf.urlban:
+        used_url = re.sub(conf.urlban, "localhost", content.url)
+    else:
+        used_url = content.url
+
+    print()
+    print("Exiting after prep due to --prep-only flag")
+    print()
+    print("Results under ./workingdir")
+    print("Source  (./workingdir/{})".format(content.tarball_prefix))
+    print("Name    (./workingdir/name)    :", content.name)
+    print("Version (./workingdir/version) :", content.version)
+    print("URL     (./workingdir/source0) :", used_url)
+    write_out(os.path.join(workingdir, "name"), content.name)
+    write_out(os.path.join(workingdir, "version"), content.version)
+    write_out(os.path.join(workingdir, "source0"), used_url)
 
 
 def main():
+    """Entry point for autospec."""
     parser = argparse.ArgumentParser()
     parser.add_argument("-g", "--skip-git",
                         action="store_false", dest="git", default=True,
                         help="Don't commit result to git")
-    parser.add_argument("-n", "--name", nargs=1,
-                        action="store", dest="name", default="",
+    parser.add_argument("-n", "--name", action="store", dest="name", default="",
                         help="Override the package name")
-    parser.add_argument("url",
+    parser.add_argument("-v", "--version", action="store", dest="version", default="",
+                        help="Override the package version")
+    parser.add_argument("url", default="", nargs="?",
                         help="tarball URL (e.g."
                              " http://example.com/downloads/mytar.tar.gz)")
     parser.add_argument('-a', "--archives", action="store",
@@ -144,72 +142,199 @@ def main():
                         default="/usr/share/defaults/autospec/autospec.conf",
                         help="Set configuration file to use")
     parser.add_argument("-t", "--target", dest="target", action="store",
-                        default=None,
+                        required=True,
                         help="Target location to create or reuse")
+    parser.add_argument("-i", "--integrity", action="store_true",
+                        default=False,
+                        help="Search for package signature from source URL and "
+                             "attempt to verify package")
+    parser.add_argument("-p", "--prep-only", action="store_true",
+                        default=False,
+                        help="Only perform preparatory work on package")
+    parser.add_argument("--non_interactive", action="store_true",
+                        default=False,
+                        help="Disable interactive mode for package verification")
+    parser.add_argument("-C", "--cleanup", dest="cleanup", action="store_true",
+                        default=False,
+                        help="Clean up mock chroot after building the package")
+    parser.add_argument("-m", "--mock-config", action="store", default="clear",
+                        help="Value to pass with Mock's -r option. Defaults to "
+                             "\"clear\", meaning that Mock will use "
+                             "/etc/mock/clear.cfg.")
+    parser.add_argument("-o", "--mock-opts", action="store", default="",
+                        help="Arbitrary options to pass down to mock when "
+                        "building a package.")
 
     args = parser.parse_args()
-    if len(args.archives) % 2 != 0:
+
+    name, url, archives = read_old_metadata()
+    name = args.name or name
+    url = args.url or url
+    archives = args.archives or archives
+
+    if not args.target:
         parser.error(argparse.ArgumentTypeError(
-                     "-a/--archives requires an even number of arguments"))
+            "The target option is not valid"))
+    else:
+        # target path must exist or be created
+        os.makedirs(args.target, exist_ok=True)
+
+    if not url:
+        parser.error(argparse.ArgumentTypeError(
+            "the url argument or options.conf['package']['url'] is required"))
+
+    if len(archives) % 2 != 0:
+        parser.error(argparse.ArgumentTypeError(
+            "-a/--archives or options.conf['package']['archives'] requires an "
+            "even number of arguments"))
+
+    if args.prep_only:
+        os.makedirs("workingdir", exists_ok=True)
+        package(args, url, name, archives, "./workingdir")
+    else:
+        with tempfile.TemporaryDirectory() as workingdir:
+            package(args, url, name, archives, workingdir)
+
+
+def package(args, url, name, archives, workingdir):
+    """Entry point for building a package with autospec."""
+    conf = config.Config(args.target)
+    check_requirements(args.git)
+    conf.detect_build_from_url(url)
+    package = build.Build()
+
     #
     # First, download the tarball, extract it and then do a set
     # of static analysis on the content of the tarball.
     #
-    build.setup_patterns()
+    filemanager = files.FileManager(conf, package)
+    content = tarball.Content(url, name, args.version, archives, conf, workingdir)
+    content.process(filemanager)
+    conf.content = content  # hack to avoid recursive dependency on init
+    # Search up one level from here to capture multiple versions
+    _dir = content.path
 
-    tarball.download_tarball(args.url, args.name, args.archives, args.target)
-    dir = tarball.path
-    if args.license_only:
-        try:
-            with open(os.path.join(build.download_path,
-                      tarball.name + ".license"), "r") as dotlic:
-                for word in dotlic.read().split():
-                    if word.find(":") < 0:
-                        license.add_license(word)
-        except:
-            pass
-        license.scan_for_licenses(name, dir)
+    conf.setup_patterns()
+    conf.config_file = args.config
+    requirements = buildreq.Requirements(content.url)
+    requirements.set_build_req(conf)
+    conf.parse_config_files(args.bump, filemanager, content.version, requirements)
+    conf.setup_patterns(conf.failed_pattern_dir)
+    conf.parse_existing_spec(content.name)
+
+    if args.prep_only:
+        write_prep(conf, workingdir, content)
         exit(0)
 
-    config.config_file = args.config
-    config.parse_config_files(build.download_path, args.bump)
-    config.parse_existing_spec(build.download_path, tarball.name)
+    if args.license_only:
+        try:
+            with open(os.path.join(conf.download_path,
+                                   content.name + ".license"), "r") as dotlic:
+                for word in dotlic.read().split():
+                    if ":" not in word:
+                        license.add_license(word)
+        except Exception:
+            pass
+        # Start one directory higher so we scan *all* versions for licenses
+        license.scan_for_licenses(os.path.dirname(_dir), conf, name)
+        exit(0)
 
-    buildreq.scan_for_configure(name, dir, build.download_path)
-    specdescription.scan_for_description(name, dir)
-    license.scan_for_licenses(name, dir)
-    docs.scan_for_changes(build.download_path, dir)
-    add_sources(build.download_path, args.archives)
-    test.scan_for_tests(dir)
+    requirements.scan_for_configure(_dir, content.name, conf)
+    specdescription.scan_for_description(content.name, _dir, conf.license_translations, conf.license_blacklist)
+    # Start one directory higher so we scan *all* versions for licenses
+    license.scan_for_licenses(os.path.dirname(_dir), conf, content.name)
+    commitmessage.scan_for_changes(conf.download_path, _dir, conf.transforms)
+    conf.add_sources(archives, content)
+    check.scan_for_tests(_dir, conf, requirements, content)
 
     #
     # Now, we have enough to write out a specfile, and try to build it.
     # We will then analyze the build result and learn information until the
     # package builds
     #
-    write_spec(build.download_path + "/" + tarball.name + ".spec")
+    specfile = specfiles.Specfile(content.url,
+                                  content.version,
+                                  content.name,
+                                  content.release,
+                                  conf,
+                                  requirements,
+                                  content)
+    filemanager.load_specfile(specfile)
+    load_specfile(conf, specfile)
 
     print("\n")
+
+    if args.integrity:
+        interactive_mode = not args.non_interactive
+        pkg_integrity.check(url, conf, interactive=interactive_mode)
+        pkg_integrity.load_specfile(specfile)
+
+    spec_type = specfile.write_spec()
+
     while 1:
-        build.package()
-        write_spec(build.download_path + "/" + tarball.name + ".spec")
-        files.newfiles_printed = 0
-        if build.round > 20 or build.must_restart == 0:
+        package.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, content, args.cleanup)
+        if spec_type == "template":
+            # specfile template is assumed "correct" and any failures need to be manually addressed
+            break
+        filemanager.load_specfile(specfile)
+        if 'license' in specfile.packages and not conf.config_opts['has_license']:
+            conf.config_opts['has_license'] = True
+            conf.rewrite_config_opts()
+        specfile.write_spec()
+        filemanager.newfiles_printed = 0
+        mock_chroot = "/var/lib/mock/clear-{}/root/builddir/build/BUILDROOT/" \
+                      "{}-{}-{}.x86_64".format(package.uniqueext,
+                                               content.name,
+                                               content.version,
+                                               content.release)
+        if filemanager.clean_directories(mock_chroot):
+            # directories added to the blacklist, need to re-run
+            package.must_restart += 1
+
+        if package.round > 20 or (package.must_restart == 0 and package.file_restart == 0):
             break
 
-    test.check_regression(build.download_path)
+        save_mock_logs(conf.download_path, package.round)
 
-    if build.success == 0:
-        print("Build failed")
-        return
+    if package.success == 0:
+        conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
+        print_build_failed()
+        sys.exit(1)
+    elif 'license' not in specfile.packages and conf.config_opts['has_license']:
+        print_fatal("package -license subpackage deleted")
+        conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
+        sys.exit(1)
+    elif os.path.isfile("README.clear"):
+        try:
+            print("\nREADME.clear CONTENTS")
+            print("*********************")
+            with open("README.clear", "r") as readme_f:
+                print(readme_f.read())
 
-    with open(build.download_path + "/release", "w") as fp:
-        fp.write(tarball.release + "\n")
+            print("*********************\n")
+        except Exception:
+            pass
 
-    commitmessage.guess_commit_message()
+    if spec_type == "generate":
+        check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
+
+    examine_abi(conf.download_path, content.name)
+    if os.path.exists("/var/lib/rpm"):
+        pkg_scan.get_whatrequires(content.name, conf.yum_conf)
+
+    write_out(conf.download_path + "/release", content.release + "\n")
+
+    # record logcheck output
+    logcheck(conf.download_path)
+
+    commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf, content)
+    conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
 
     if args.git:
-        git.commit_to_git(build.download_path)
+        git.commit_to_git(conf, content.name, package.success)
+    else:
+        print("To commit your changes, git add the relevant files and "
+              "run 'git commit -F commitmsg'")
 
 
 if __name__ == '__main__':

autospec/buildpattern.py

@@ -1,379 +0,0 @@
-#!/bin/true
-#
-# buildpattern.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Deduce and emit the patterns for %build
-#
-
-import config
-import files
-import lang
-import os
-import patches
-import re
-import tarball
-import test
-
-default_pattern = "make"
-pattern_strengh = 0
-
-disable_static = "--disable-static"
-
-extra_make = ""
-extra_cmake = ""
-extra_make_install = ""
-install_macro = "%make_install"
-make_install_append = []
-subdir = ""
-sources = {"unit": [], "gcov": [], "tmpfile": [], "archive": []}
-source_index = {}
-archive_details = {}
-
-
-def write_prep(file, ruby_pattern=False):
-    file.write_strip("%prep")
-    for archive in sources["archive"]:
-        file.write_strip("tar -xf %{{SOURCE{}}}".format(source_index[archive]))
-        file.write_strip("cd ..")
-    if ruby_pattern:
-        file.write_strip("gem unpack %{SOURCE0}")
-        file.write_strip("%setup -q -D -T -n " + tarball.tarball_prefix)
-        file.write_strip("gem spec %{SOURCE0} -l --ruby > " + tarball.name + ".gemspec")
-    else:
-        if default_pattern == 'R':
-            file.write_strip("%setup -q -c -n " + tarball.tarball_prefix)
-        else:
-            file.write_strip("%setup -q -n " + tarball.tarball_prefix)
-    for archive in sources["archive"]:
-        file.write_strip('mv %{{_topdir}}/BUILD/{0}/* ./{1}'
-                         .format(archive_details[archive + "prefix"],
-                                 archive_details[archive + "destination"]))
-    patches.apply_patches(file)
-    file.write_strip("\n")
-
-
-def write_variables(file):
-    flags = []
-    if config.optimize_size:
-        flags.extend(["-Os", "-ffunction-sections"])
-    if config.insecure_build:
-        file.write_strip("export CFLAGS=\"-O2 -g\"\n")
-        file.write_strip("unset LDFLAGS\n")
-    if config.optimize_speed:
-        flags.extend(["-O3", "-flto", "-ffunction-sections", "-fno-semantic-interposition"])
-        file.write_strip("export AR=gcc-ar\n")
-        file.write_strip("export RANLIB=gcc-ranlib\n")
-    if tarball.gcov_file:
-        flags = list(filter(("-flto").__ne__, flags))
-        flags.extend(["-O3", "-fauto-profile=%{{SOURCE{0}}}".format(source_index[sources["gcov"][0]])])
-    if flags:
-        flags = list(set(flags))
-        file.write_strip("export CFLAGS=\"$CFLAGS {0} \"\n".format(" ".join(flags)))
-        file.write_strip("export CXXFLAGS=\"$CXXFLAGS {0} \"\n".format(" ".join(flags)))
-
-
-def write_check(file):
-    if test.tests_config:
-        file.write_strip("%check")
-        file.write_strip("export http_proxy=http://127.0.0.1:9/")
-        file.write_strip("export https_proxy=http://127.0.0.1:9/")
-        file.write_strip("export no_proxy=localhost")
-        file.write_strip(test.tests_config)
-        file.write_strip("\n")
-
-
-def write_make_install(file):
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("%s %s\n" % (install_macro, extra_make_install))
-    if subdir:
-        file.write_strip("popd")
-    lang.write_find_lang(file)
-
-
-def write_configure_pattern(file):
-    if patches.autoreconf:
-        # Patches affecting configure.* or Makefile.*, reconf instead
-        write_configure_ac_pattern(file)
-        return
-    write_prep(file)
-    file.write_strip("%build")
-    if config.asneeded == 0:
-        file.write_strip("unset LD_AS_NEEDED\n")
-    write_variables(file)
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("%configure " + disable_static + " " + config.extra_configure)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    if subdir:
-        file.write_strip("popd")
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_configure_ac_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    if config.asneeded == 0:
-        file.write_strip("unset LD_AS_NEEDED\n")
-    write_variables(file)
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("%reconfigure " + disable_static + " " + config.extra_configure)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    if subdir:
-        file.write_strip("popd")
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_make_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_autogen_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("%autogen " + disable_static + " " + config.extra_configure)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_distutils_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("python2 setup.py build -b py2 " + config.extra_configure)
-    file.write_strip("\n")
-    if test.tests_config:
-        file.write_strip("%check")
-        # Prevent setuptools from hitting the internet
-        file.write_strip("export http_proxy=http://127.0.0.1:9/")
-        file.write_strip("export https_proxy=http://127.0.0.1:9/")
-        file.write_strip("export no_proxy=localhost,127.0.0.1,0.0.0.0")
-        file.write_strip(test.tests_config)
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("python2 -tt setup.py build -b py2 install --root=%{buildroot}")
-    lang.write_find_lang(file)
-
-
-def write_distutils23_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("python2 setup.py build -b py2 " + config.extra_configure)
-    file.write_strip("python3 setup.py build -b py3 " + config.extra_configure)
-    file.write_strip("\n")
-    if test.tests_config:
-        file.write_strip("%check")
-        # Prevent setuptools from hitting the internet
-        file.write_strip("export http_proxy=http://127.0.0.1:9/")
-        file.write_strip("export https_proxy=http://127.0.0.1:9/")
-        file.write_strip("export no_proxy=localhost,127.0.0.1,0.0.0.0")
-        file.write_strip(test.tests_config)
-
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("python2 -tt setup.py build -b py2 install --root=%{buildroot}")
-    file.write_strip("python3 -tt setup.py build -b py3 install --root=%{buildroot}")
-    lang.write_find_lang(file)
-
-
-def write_R_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    file.write_strip("\n")
-
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("export LANG=C")
-    file.write_strip("export CFLAGS=\"$CFLAGS -O3 -flto -ffunction-sections -fno-semantic-interposition \"\n")
-    file.write_strip("export CXXFLAGS=\"$CXXFLAGS -O3 -flto -ffunction-sections -fno-semantic-interposition \"\n")
-    file.write_strip("export AR=gcc-ar\n")
-    file.write_strip("export RANLIB=gcc-ranlib\n")
-    file.write_strip("export LDFLAGS=\"$LDFLAGS  -Wl,-z -Wl,relro\"\n")
-
-    file.write_strip("mkdir -p %{buildroot}/usr/lib64/R/library")
-    file.write_strip("R CMD INSTALL --install-tests --build  -l %{buildroot}/usr/lib64/R/library " + tarball.rawname)
-    file.write_strip("%{__rm} -rf %{buildroot}%{_datadir}/R/library/R.css")
-    lang.write_find_lang(file)
-    write_check(file)
-
-
-def write_ruby_pattern(file):
-    write_prep(file, ruby_pattern=True)
-    file.write_strip("%build")
-    file.write_strip("gem build " + tarball.name + ".gemspec")
-    file.write_strip("\n")
-
-    file.write_strip("%install")
-    file.write_strip("%global gem_dir $(ruby -e'puts Gem.default_dir')")
-    file.write_strip("gem install -V \\")
-    file.write_strip("  --local \\")
-    file.write_strip("  --force \\")
-    file.write_strip("  --install-dir .%{gem_dir} \\")
-    file.write_strip("  --bindir .%{_bindir} \\")
-    file.write_strip(" " + tarball.tarball_prefix + ".gem")
-    file.write_strip("\n")
-
-    file.write_strip("mkdir -p %{buildroot}%{gem_dir}")
-    file.write_strip("cp -pa .%{gem_dir}/* \\")
-    file.write_strip("        %{buildroot}%{gem_dir}")
-    file.write_strip("\n")
-
-    file.write_strip("if [ -d .%{_bindir} ]; then")
-    file.write_strip("    mkdir -p %{buildroot}%{_bindir}")
-    file.write_strip("    cp -pa .%{_bindir}/* \\")
-    file.write_strip("        %{buildroot}%{_bindir}/")
-    file.write_strip("fi")
-    file.write_strip("\n")
-    lang.write_find_lang(file)
-    write_check(file)
-
-
-def write_cmake_pattern(file):
-    global subdir
-    subdir = "clr-build"
-    write_prep(file)
-    file.write_strip("%build")
-    file.write_strip("mkdir clr-build")
-    file.write_strip("pushd clr-build")
-    file.write_strip("cmake .. -G \"Unix Makefiles\" -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS:BOOL=ON -DLIB_INSTALL_DIR:PATH=%{_libdir} " + extra_cmake)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("popd")
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_cpan_pattern(file):
-    global subdir
-    write_prep(file)
-    file.write_strip("%build")
-    file.write_strip("if test -f Makefile.PL; then")
-    file.write_strip("%{__perl} Makefile.PL")
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("else")
-    file.write_strip("%{__perl} Build.PL")
-    file.write_strip("./Build")
-    file.write_strip("fi")
-    file.write_strip("\n")
-    write_check(file)
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("if test -f Makefile.PL; then")
-    file.write_strip("make pure_install PERL_INSTALL_ROOT=%{buildroot}")
-    file.write_strip("else")
-    file.write_strip("./Build install --installdirs=site --destdir=%{buildroot}")
-    file.write_strip("fi")
-    file.write_strip("find %{buildroot} -type f -name .packlist -exec rm -f {} ';'")
-    file.write_strip("find %{buildroot} -depth -type d -exec rmdir {} 2>/dev/null ';'")
-    file.write_strip("find %{buildroot} -type f -name '*.bs' -empty -exec rm -f {} ';'")
-    file.write_strip("%{_fixperms} %{buildroot}/*")
-    lang.write_find_lang(file)
-
-
-def write_scons_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("scons" + config.parallel_build + " " + config.extra_configure)
-    file.write_strip("\n")
-    file.write_strip("%install")
-    file.write_strip("scons install " + extra_make_install)
-
-
-def set_build_pattern(pattern, strength):
-    global default_pattern
-    global pattern_strengh
-    if strength <= pattern_strengh:
-        return 0
-    default_pattern = pattern
-    pattern_strengh = strength
-
-
-def get_systemd_units():
-    """get systemd unit files from the files module"""
-    service_file_section = "config"
-    systemd_service_pattern = r"^/usr/lib/systemd/system/[^/]*\.(mount|service|socket|target)$"
-    systemd_units = []
-
-    if service_file_section not in files.packages:
-        return systemd_units
-
-    for f in files.packages[service_file_section]:
-        if re.search(systemd_service_pattern, f) and f not in files.excludes:
-            systemd_units.append(f)
-
-    return systemd_units
-
-
-def write_sources(file):
-    """write out installs from SourceX lines"""
-    if len(sources["unit"]) != 0:
-        file.write_strip("mkdir -p %{buildroot}/usr/lib/systemd/system")
-        for unit in sources["unit"]:
-            file.write_strip("install -m 0644 %{{SOURCE{0}}} %{{buildroot}}/usr/lib/systemd/system/{1}"
-                             .format(source_index[unit], unit))
-    if len(sources["tmpfile"]) != 0:
-        file.write_strip("mkdir -p %{buildroot}/usr/lib/tmpfiles.d")
-        file.write_strip("install -m 0644 %{{SOURCE{0}}} %{{buildroot}}/usr/lib/tmpfiles.d/{1}.conf"
-                         .format(source_index[sources["tmpfile"][0]], tarball.name))
-
-
-def write_make_install_append(file):
-    """write out any custom supplied commands at the very end of the %install section"""
-    if make_install_append and make_install_append[0]:
-        file.write_strip("## make_install_append content")
-        for line in make_install_append:
-            file.write_strip("%s\n" % line)
-        file.write_strip("## make_install_append end")
-
-
-def write_systemd_units(file):
-    """write out installs for systemd unit files"""
-    units = get_systemd_units()
-    for unit in units:
-        file.write("systemctl --root=%{{buildroot}} enable {0}\n".format(os.path.basename(unit)))
-
-
-def write_buildpattern(file):
-    file.write_strip("\n")
-
-    pattern_method = globals().get('write_%s_pattern' % default_pattern, None)
-    if pattern_method:
-        pattern_method(file)
-
-    write_sources(file)
-    write_make_install_append(file)
-    # write_systemd_units(file)

autospec/check.py

@@ -0,0 +1,175 @@
+#!/bin/true
+#
+# test.py - part of autospec
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Deduce and emmit the patterns for %check
+#
+
+import os
+import re
+
+import count
+import util
+
+tests_config = ""
+
+
+def check_regression(pkg_dir, skip_tests, test_round):
+    """Check the build log for test regressions using the count module."""
+    if skip_tests:
+        return
+
+    log_path = os.path.join(pkg_dir, 'results', 'build.log')
+    result = count.parse_log(log_path)
+    if len(result) == 0 or result[0:2] == ',0':
+        log_path = os.path.join(pkg_dir, 'results', f"round{test_round}-build.log")
+        result = count.parse_log(log_path)
+
+    titles = [('Package', 'package name', 1),
+              ('Total', 'total tests', 1),
+              ('Pass', 'total passing', 1),
+              ('Fail', 'total failing', 0),
+              ('Skip', 'tests skipped', 0),
+              ('XFail', 'expected fail', 0)]
+    res_str = ""
+    for line in result.strip('\n').split('\n'):
+        s_line = line.split(',')
+        for idx, title in enumerate(titles):
+            if s_line[idx]:
+                if (s_line[idx] != '0') or (title[2] > 0):
+                    print("{}: {}".format(title[1], s_line[idx]))
+                res_str += "{} : {}\n".format(title[0], s_line[idx])
+
+    util.write_out(os.path.join(pkg_dir, "testresults"), res_str)
+
+
+def scan_for_tests(src_dir, config, requirements, content):
+    """Scan source directory for test files and set tests_config accordingly."""
+    global tests_config
+
+    if config.config_opts.get('skip_tests') or tests_config:
+        return
+
+    make_command = "ninja" if config.config_opts.get('use_ninja') else "make"
+    makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
+    make_check = "{} {}check".format(make_command, makeflags)
+    cmake_check = "{} test".format(make_command)
+    make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                         "{} {}check\nmodule unload openmpi".format(make_command, makeflags)
+    cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                          "{} test\nmodule unload openmpi".format(make_command)
+
+    if config.config_opts.get('allow_test_failures'):
+        make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                             "{} {}check || :\nmodule unload openmpi".format(make_command, makeflags)
+        cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                              "{} test || :\nmodule unload openmpi".format(make_command)
+
+    perl_check = "{} TEST_VERBOSE=1 test".format(make_command)
+    meson_check = "meson test -C builddir --print-errorlogs"
+    if config.config_opts.get('allow_test_failures'):
+        make_check += " || :"
+        cmake_check += " || :"
+        perl_check += " || :"
+        meson_check += " || :"
+
+    testsuites = {
+        "makecheck": make_check,
+        "perlcheck": perl_check,
+        "cmake": "cd clr-build; " + cmake_check,
+        "meson": meson_check,
+    }
+    if config.config_opts.get('32bit'):
+        testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../build32/clr-build32;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
+    if config.config_opts.get('use_avx2'):
+        testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx2/clr-build-avx2;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx2;\n" + meson_check + " || :"
+    if config.config_opts.get('use_avx512'):
+        testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildavx512/clr-build-avx512;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildavx512;\n" + meson_check + " || :"
+    if config.config_opts.get('use_apx'):
+        testsuites["makecheck"] += "\ncd ../buildapx;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../../buildapx/clr-build-apx;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../buildapx;\n" + meson_check + " || :"
+    if config.config_opts.get('openmpi'):
+        testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
+        testsuites["cmake"] += "\ncd ../../build-openmpi/clr-build-openmpi;\n" + cmake_check_openmpi
+
+    files = os.listdir(src_dir)
+
+    if config.default_pattern == "cmake":
+        makefile_path = os.path.join(src_dir, "CMakeLists.txt")
+        if not os.path.isfile(makefile_path):
+            return
+
+        if "enable_testing" in util.open_auto(makefile_path).read():
+            tests_config = testsuites["cmake"]
+
+    elif config.default_pattern in ["cpan", "configure", "configure_ac", "autogen"] and "Makefile.in" in files:
+        makefile_path = os.path.join(src_dir, "Makefile.in")
+        if os.path.isfile(makefile_path):
+            with util.open_auto(makefile_path, 'r') as make_fp:
+                lines = make_fp.readlines()
+            for line in lines:
+                if line.startswith("check:"):
+                    tests_config = testsuites["makecheck"]
+                    break
+                if line.startswith("test:"):
+                    tests_config = testsuites["perlcheck"]
+                    break
+
+    elif config.default_pattern in ["configure", "configure_ac", "autogen"] and "Makefile.am" in files:
+        tests_config = testsuites["makecheck"]
+
+    elif config.default_pattern in ["cpan"] and "Makefile.PL" in files:
+        tests_config = testsuites["perlcheck"]
+
+    elif config.default_pattern == "R":
+        tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n"              \
+                       "R CMD check --no-manual --no-examples --no-codoc . "    \
+                       "|| :"
+    elif config.default_pattern == "meson":
+        found_tests = False
+        makefile_path = os.path.join(src_dir, "meson.build")
+        if not os.path.isfile(makefile_path):
+            return
+        for dirpath, _, files in os.walk(src_dir):
+            for f in files:
+                if f == "meson.build":
+                    with util.open_auto(os.path.join(dirpath, f)) as fp:
+                        if any(re.search(r'^\s*test\s*\(.+', line) for line in fp):
+                            found_tests = True
+                            tests_config = testsuites["meson"]
+                            break
+            if found_tests:
+                break
+
+    if "tox.ini" in files:
+        requirements.add_buildreq("pypi-tox")
+        requirements.add_buildreq("pypi-pytest")
+        requirements.add_buildreq("pypi-virtualenv")
+        requirements.add_buildreq("pypi-pluggy")
+        requirements.add_buildreq("pypi(py)")
+
+
+def load_specfile(specfile):
+    """Load the specfile object."""
+    specfile.tests_config = tests_config

autospec/cmake_modules

@@ -0,0 +1,972 @@
+# This file maps the CMake FIND_PACKAGE name to the packaage name, to be
+# added to BuildRequires. Please keep the list sorted
+# Format: <cmake modulename>, <pkg name(s)>
+2Geom, inkscape-dev
+ALSA, alsa-lib-dev
+ASPELL, aspell-dev
+ATen, pypi(pytorch)
+AccountsQt5, libaccounts-qt-dev
+Alembic, scene-alembic-dev
+Analitza5, analitza-dev
+AppStreamQt, appstream-dev
+Arcus, libArcus-dev
+Argon2, argon2-dev
+Armadillo, armadillo-data
+Astro, marble-dev
+BISON, bison-dev
+BLAS, openblas
+BZip2, bzip2-dev
+Backtrace, glibc-dev
+Boost, boost-dev
+Breeze, breeze-dev
+CAF, zeek-dev
+CGAL, CGAL-dev
+CGALLib, CGAL-dev
+CGAL_CoreLib, CGAL-dev
+CGAL_ImageIOLib, CGAL-dev
+CGAL_Qt5Lib, CGAL-dev
+CLucene, clucene-core
+CURL, curl-dev
+Caffe2, pypi(pytorch)
+Cantor, cantor-dev
+CapnProto, capnproto-dev
+CheckFlagCommon, cmake-data
+Clang, llvm-dev
+Codec2, codec2-dev
+CommonCMake, zeek-data
+Cups, cups-dev
+Curses, ncurses-dev
+DBus1, dbus-dev
+DigikamCore, digikam-dev
+DigikamDatabase, digikam-dev
+DigikamGui, digikam-dev
+DigikamPlugin, digikam-dev
+DolphinVcs, dolphin-dev
+Doxygen, doxygen
+ECM, extra-cmake-modules-data
+EGL, extra-cmake-modules pkgconfig(egl)
+EXPAT, expat-dev
+Ecore, efl-dev
+EcoreCxx, efl-dev
+Edje, efl-dev
+Eet, efl-dev
+EetCxx, efl-dev
+Eeze, efl-dev
+Efl, efl-dev
+Efreet, efl-dev
+Eigen3, eigen-data
+Eina, efl-dev
+EinaCxx, efl-dev
+Eio, efl-dev
+Eldbus, efl-dev
+Elementary, efl-dev
+Elua, efl-dev
+Emile, efl-dev
+Emotion, efl-dev
+Eo, efl-dev
+EoCxx, efl-dev
+Eolian, efl-dev
+EolianCxx, efl-dev
+Ethumb, efl-dev
+EthumbClient, efl-dev
+Evas, efl-dev
+EvasCxx, efl-dev
+FLEX, flex
+FLTK, fltk-dev
+FindFont, VTK-dev
+FindPkg, cmake-data
+FreeGLUT, freeglut-dev
+FreeRDP, FreeRDP-dev
+FreeRDP-Client, FreeRDP-dev
+FreeRDP-Server, FreeRDP-dev
+FreeRDP-Shadow, FreeRDP-dev
+Freetype, freetype-dev
+GDAL, gdal-dev
+GLEW, glew-dev
+GLIB2, extra-cmake-modules pkgconfig(glib-2.0)
+GLU, glu-dev
+GLUT, freeglut-dev
+GSL, gsl-dev
+GTK2, gtk+-dev
+GTest, googletest-dev
+Gcrypt, libgcrypt-dev
+Gettext, gettext-dev
+Git, git
+Gloo, pypi(pytorch)
+Gmic, gmic-dev
+GnuTLS, gnutls-dev
+Gnuplot, gnuplot-dev
+Gperf, extra-cmake-modules gperf
+Gpgmepp, gpgme-dev
+Grantlee5, grantlee-dev
+GtkDoc, gtk-doc-data
+HDF5, hdf5-dev
+ICU, icu4c-dev
+Iconv, glibc-dev
+IlmBase, openexr-dev
+Imath, Imath-dev
+InferenceEngine, dldt-dev
+Inotify, extra-cmake-modules
+Intl, glibc-dev
+JNI, openjdk-dev
+JPEG, libjpeg-turbo-dev
+Java, openjdk
+KAccounts, kaccounts-integration-dev
+KChart, kdiagram-dev
+KDED, kded-dev
+KDEExperimentalPurpose, purpose-dev
+KDSoap, kdsoap-dev
+KDecoration2, kdecoration-dev
+KF5Activities, kactivities-dev
+KF5ActivitiesStats, kactivities-stats-dev
+KF5Akonadi, akonadi-dev
+KF5AkonadiCalendar, akonadi-calendar-dev
+KF5AkonadiContact, akonadi-contacts-dev
+KF5AkonadiMime, akonadi-mime-dev
+KF5AkonadiNotes, akonadi-notes-dev
+KF5AkonadiSearch, akonadi-search-dev
+KF5AlarmCalendar, kalarmcal-dev
+KF5Archive, karchive-dev
+KF5Attica, attica-dev
+KF5Auth, kauth-dev
+KF5Baloo, baloo-dev
+KF5BalooWidgets, baloo-widgets-dev
+KF5Blog, kblog-dev
+KF5BluezQt, bluez-qt-dev
+KF5Bookmarks, kbookmarks-dev
+KF5CalendarCore, kcalendarcore-dev
+KF5CalendarSupport, calendarsupport-dev
+KF5CalendarUtils, kcalutils-dev
+KF5Cddb, libkcddb-dev
+KF5Codecs, kcodecs-dev
+KF5CompactDisc, libkcompactdisc-dev
+KF5Completion, kcompletion-dev
+KF5Config, kconfig-dev
+KF5ConfigWidgets, kconfigwidgets-dev
+KF5ContactEditor, akonadi-contacts-dev
+KF5Contacts, kcontacts-dev
+KF5CoreAddons, kcoreaddons-dev
+KF5Crash, kcrash-dev
+KF5DAV, kdav-dev
+KF5DBusAddons, kdbusaddons-dev
+KF5DNSSD, kdnssd-dev
+KF5Declarative, kdeclarative-dev
+KF5DesignerPlugin, kdesignerplugin-dev
+KF5DocTools, kdoctools-dev
+KF5Emoticons, kemoticons-dev
+KF5EventViews, eventviews-dev
+KF5FileMetaData, kfilemetadata-dev
+KF5FollowupReminder, kdepim-apps-libs-dev
+KF5FrameworkIntegration, frameworkintegration-dev
+KF5GlobalAccel, kglobalaccel-dev
+KF5GrantleeTheme, grantleetheme-dev
+KF5Gravatar, libgravatar-dev
+KF5GuiAddons, kguiaddons-dev
+KF5Holidays, kholidays-dev
+KF5I18n, ki18n-dev
+KF5IMAP, kimap-dev
+KF5IconThemes, kiconthemes-dev
+KF5IdentityManagement, kidentitymanagement-dev
+KF5IdleTime, kidletime-dev
+KF5Init, kinit-dev
+KF5ItemModels, kitemmodels-dev
+KF5ItemViews, kitemviews-dev
+KF5JS, kjs-dev
+KF5JobWidgets, kjobwidgets-dev
+KF5JsEmbed, kjsembed-dev
+KF5KCMUtils, kcmutils-dev
+KF5KDE4Support, kdelibs4support-dev
+KF5KDEGames, libkdegames-dev
+KF5KDELibs4Support, kdelibs4support-dev
+KF5KDcraw, libkdcraw-dev
+KF5KExiv2, libkexiv2-dev
+KF5KHtml, khtml-dev
+KF5KIO, kio-dev
+KF5KMahjongglib, libkmahjongg-dev
+KF5KaddressbookGrantlee, kdepim-apps-libs-dev
+KF5KaddressbookImportExport, kdepim-apps-libs-dev
+KF5KdepimDBusInterfaces, kdepim-apps-libs-dev
+KF5Kipi, libkipi-dev
+KF5Kirigami2, kirigami2-dev
+KF5Konq, konqueror-dev
+KF5KontactInterface, kontactinterface-dev
+KF5Kross, kross-dev
+KF5Ldap, kldap-dev
+KF5LibKSieve, libksieve-dev
+KF5Libkdepim, libkdepim-dev
+KF5LibkdepimAkonadi, libkdepim-dev
+KF5Libkleo, libkleo-dev
+KF5MailCommon, mailcommon-dev
+KF5MailImporter, mailimporter-dev
+KF5MailImporterAkonadi, mailimporter-dev
+KF5MailTransport, kmailtransport-dev
+KF5MailTransportAkonadi, kmailtransport-dev
+KF5Mbox, kmbox-dev
+KF5MediaPlayer, kmediaplayer-dev
+KF5MessageComposer, messagelib-dev
+KF5MessageCore, messagelib-dev
+KF5MessageList, messagelib-dev
+KF5MessageViewer, messagelib-dev
+KF5Mime, kmime-dev
+KF5MimeTreeParser, messagelib-dev
+KF5ModemManagerQt, modemmanager-qt-dev
+KF5NetworkManagerQt, networkmanager-qt-dev
+KF5NewStuff, knewstuff-dev
+KF5NewStuffCore, knewstuff-dev
+KF5NewStuffQuick, knewstuff-dev
+KF5Notifications, knotifications-dev
+KF5NotifyConfig, knotifyconfig-dev
+KF5Package, kpackage-dev
+KF5Parts, kparts-dev
+KF5People, kpeople-dev
+KF5PimCommon, pimcommon-dev
+KF5PimCommonAkonadi, pimcommon-dev
+KF5PimTextEdit, kpimtextedit-dev
+KF5Plasma, plasma-framework-dev
+KF5PlasmaQuick, plasma-framework-dev
+KF5Plotting, kplotting-dev
+KF5Prison, prison-dev
+KF5Pty, kpty-dev
+KF5Purpose, purpose-dev
+KF5QQC2DeskopStyle, qqc2-desktop-style-dev
+KF5QQC2DesktopStyle, qqc2-desktop-style-dev
+KF5QuickCharts, kquickcharts-dev
+KF5Runner, krunner-dev
+KF5Sane, libksane-dev
+KF5Screen, libkscreen-dev
+KF5SendLater, kdepim-apps-libs-dev
+KF5Service, kservice-dev
+KF5Solid, solid-dev
+KF5Sonnet, sonnet-dev
+KF5Su, kdesu-dev
+KF5Syndication, syndication-dev
+KF5SyntaxHighlighting, syntax-highlighting-dev
+KF5SysGuard, libksysguard-dev
+KF5TemplateParser, messagelib-dev
+KF5TextEditor, ktexteditor-dev
+KF5TextWidgets, ktextwidgets-dev
+KF5ThreadWeaver, threadweaver-dev
+KF5Tnef, ktnef-dev
+KF5UnitConversion, kunitconversion-dev
+KF5Wallet, kwallet-dev
+KF5Wayland, kwayland-dev
+KF5WebEngineViewer, messagelib-dev
+KF5WidgetsAddons, kwidgetsaddons-dev
+KF5WindowSystem, kwindowsystem-dev
+KF5XmlGui, kxmlgui-dev
+KF5XmlRpcClient, kxmlrpcclient-dev
+KGantt, kdiagram-dev
+KHotKeysDBusInterface, khotkeys-dev
+KPimGAPI, libkgapi-dev
+KPimItinerary, kitinerary-dev
+KPimPkPass, kpkpass-dev
+KPimSMTP, ksmtp-dev
+KRunnerAppDBusInterface, plasma-workspace-dev
+KSMServerDBusInterface, plasma-workspace-dev
+KSaneCore, ksanecore-dev
+KScreenLocker, kscreenlocker-dev
+KSysGuard, libksysguard-dev
+KUserFeedback, kuserfeedback-dev
+KWaylandServer, kwayland-server-dev
+KWinDBusInterface, kwin-dev
+KWinEffects, kwin-dev
+KastenControllers, okteta-dev
+KastenCore, okteta-dev
+KastenGui, okteta-dev
+KioArchive, kio-extras-dev
+LATEX, texlive
+LLD, llvm-dev
+LLVM, llvm-dev
+LLVM-, llvm-dev
+LayerShellQt, layer-shell-qt-dev
+Leptonica, leptonica-dev
+LibArchive, libarchive-dev
+LibColorCorrect, plasma-workspace-dev
+LibFTDI1, libftdi1-dev
+LibGPGError, libgpg-error-dev
+LibGit2, extra-cmake-modules pkgconfig(libgit2)
+LibIcal, libical-dev
+LibKEduVocDocument, libkeduvocdocument-dev
+LibKWorkspace, plasma-workspace-dev
+LibKompareDiff2, libkomparediff2-dev
+LibNotificationManager, plasma-workspace-dev
+LibObs, obs-studio-dev
+LibTaskManager, plasma-workspace-dev
+LibXml2, libxml2-dev
+LibXslt, libxslt-dev
+LibtorrentRasterbar, libtorrent-rasterbar-dev
+Lua, lua-dev
+MPI, openmpi-dev
+MailTransportDBusService, libkdepim-dev
+Marble, marble-dev
+Mlt7, mlt-dev
+Motif, motif-dev
+NLopt, nlopt-dev
+OCE, OpenCASCADE
+OktetaCore, okteta-dev
+OktetaGui, okteta-dev
+OktetaKastenControllers, okteta-dev
+OktetaKastenCore, okteta-dev
+OktetaKastenGui, okteta-dev
+Okular5, okular-dev
+OpenAL, openal-soft-dev
+OpenBLAS, openblas-dev
+OpenCV, opencv-dev
+OpenColorIO, OpenColorIO-data
+OpenEXR, extra-cmake-modules pkgconfig(OpenEXR)
+OpenGL, mesa-dev
+OpenImageIO, oiio-dev
+OpenJPEG, openjpeg
+OpenMP, cmake
+OpenSSL, openssl-dev
+PHP4, php-dev
+PNG, libpng-dev
+PahoMqttCpp, paho.mqtt.cpp-dev
+Pala, palapeli-dev
+Patch, patch
+Perl, perl
+PerlLibs, perl
+Phonon4Qt5, phonon-dev
+Phonon4Qt5Experimental, phonon-dev
+PkgConfig, pkg-config
+PlasmaActivities, plasma-activities-dev
+PlasmaPotdProvider, kdeplasma-addons-dev
+PlasmaWaylandProtocols, plasma-wayland-protocols-dev
+Png2Ico, extra-cmake-modules png2ico
+PolkitQt5-1, polkit-qt-dev
+Polly, llvm-dev
+Poppler, extra-cmake-modules pkgconfig(poppler)
+PortMidi, portmidi-dev
+PostgreSQL, postgresql-dev
+Protobuf, protobuf-dev
+PulseAudio, extra-cmake-modules pkgconfig(libpulse)
+PyIlmBase, openexr-dev
+PySide2, pyside2-setup-dev
+Python, python3
+Python3, python3-dev
+PythonInterp, python3
+PythonLibs, python3-dev
+QGpgme, gpgme-dev gpgme-extras
+QHelpGenerator, extra-cmake-modules qt6tools-dev
+QMobipocket, kdegraphics-mobipocket-dev
+Qca-qt5, qca-qt5-dev
+Qt3DTests, qt6base-dev
+Qt5, qtbase-dev mesa-dev
+Qt53DAnimation, qt3d-dev
+Qt53DCore, qt3d-dev
+Qt53DExtras, qt3d-dev
+Qt53DInput, qt3d-dev
+Qt53DLogic, qt3d-dev
+Qt53DQuick, qt3d-dev
+Qt53DQuickAnimation, qt3d-dev
+Qt53DQuickExtras, qt3d-dev
+Qt53DQuickInput, qt3d-dev
+Qt53DQuickRender, qt3d-dev
+Qt53DQuickScene2D, qt3d-dev
+Qt53DRender, qt3d-dev
+Qt5AccessibilitySupport, qtbase-dev
+Qt5AttributionsScannerTools, qttools-dev
+Qt5Bluetooth, qtconnectivity-dev
+Qt5Bootstrap, qtbase-dev
+Qt5Charts, qtcharts-dev
+Qt5CompatTests, qt6base-dev
+Qt5Concurrent, qtbase-dev
+Qt5Core, qtbase-dev
+Qt5DBus, qtbase-dev
+Qt5DataVisualization, qtdatavis3d-dev
+Qt5Designer, qttools-dev
+Qt5DesignerComponents, qttools-dev
+Qt5DeviceDiscoverySupport, qtbase-dev
+Qt5EdidSupport, qtbase-dev
+Qt5EglFSDeviceIntegration, qtbase-dev
+Qt5EglFsKmsSupport, qtbase-dev
+Qt5EglSupport, qtbase-dev
+Qt5EventDispatcherSupport, qtbase-dev
+Qt5FbSupport, qtbase-dev
+Qt5FontDatabaseSupport, qtbase-dev
+Qt5Gamepad, qtgamepad-dev
+Qt5GlxSupport, qtbase-dev
+Qt5Gui, qtbase-dev
+Qt5Help, qttools-dev
+Qt5InputSupport, qtbase-dev
+Qt5Keychain, qtkeychain-dev
+Qt5KmsSupport, qtbase-dev
+Qt5LinguistTools, qttools-dev
+Qt5Location, qtlocation-dev
+Qt5Mqtt, qtmqtt-dev
+Qt5Multimedia, qtmultimedia-dev
+Qt5MultimediaGstTools, qtmultimedia-dev
+Qt5MultimediaQuick, qtmultimedia-dev
+Qt5MultimediaWidgets, qtmultimedia-dev
+Qt5Network, qtbase-dev
+Qt5NetworkAuth, qtnetworkauth-dev
+Qt5Nfc, qtconnectivity-dev
+Qt5OpenGL, qtbase-dev
+Qt5OpenGLExtensions, qtbase-dev
+Qt5PacketProtocol, qtdeclarative-dev
+Qt5Pdf, qtwebengine-dev
+Qt5PdfWidgets, qtwebengine-dev
+Qt5PlatformCompositorSupport, qtbase-dev
+Qt5Positioning, qtlocation-dev
+Qt5PositioningQuick, qtlocation-dev
+Qt5PrintSupport, qtbase-dev
+Qt5Qml, qtdeclarative-dev
+Qt5QmlDebug, qtdeclarative-dev
+Qt5QmlDevTools, qtdeclarative-dev
+Qt5QmlImportScanner, qtdeclarative-dev
+Qt5QmlModels, qtdeclarative-dev
+Qt5QmlWorkerScript, qtdeclarative-dev
+Qt5Quick, qtdeclarative-dev
+Qt5QuickCompiler, qtdeclarative-dev
+Qt5QuickControls2, qtquickcontrols2-dev
+Qt5QuickParticles, qtdeclarative-dev
+Qt5QuickShapes, qtdeclarative-dev
+Qt5QuickTemplates2, qtquickcontrols2-dev
+Qt5QuickTest, qtdeclarative-dev
+Qt5QuickWidgets, qtdeclarative-dev
+Qt5RemoteObjects, qtremoteobjects-dev
+Qt5RepParser, qtremoteobjects-dev
+Qt5Script, qtscript-dev
+Qt5ScriptTools, qtscript-dev
+Qt5Scxml, qtscxml-dev
+Qt5Sensors, qtsensors-dev
+Qt5SerialBus, qtserialbus-dev
+Qt5SerialPort, qtserialport-dev
+Qt5ServiceSupport, qtbase-dev
+Qt5Sql, qtbase-dev
+Qt5Svg, qtsvg-dev
+Qt5Test, qtbase-dev
+Qt5TextToSpeech, qtspeech-dev
+Qt5ThemeSupport, qtbase-dev
+Qt5UiPlugin, qttools-dev
+Qt5UiTools, qttools-dev
+Qt5VirtualKeyboard, qtvirtualkeyboard-dev
+Qt5VulkanSupport, qtbase-dev
+Qt5WaylandClient, qtwayland-dev
+Qt5WaylandCompositor, qtwayland-dev
+Qt5WebChannel, qtwebchannel-dev
+Qt5WebEngine, qtwebengine-dev
+Qt5WebEngineCore, qtwebengine-dev
+Qt5WebEngineWidgets, qtwebengine-dev
+Qt5WebSockets, qtwebsockets-dev
+Qt5Widgets, qtbase-dev
+Qt5X11Extras, qtx11extras-dev
+Qt5XcbQpa, qtbase-dev
+Qt5XkbCommonSupport, qtbase-dev
+Qt5Xml, qtbase-dev
+Qt5XmlPatterns, qtxmlpatterns-dev
+Qt6, qt6base-dev
+Qt63DAnimation, qt6base-dev
+Qt63DCore, qt6base-dev
+Qt63DExtras, qt6base-dev
+Qt63DInput, qt6base-dev
+Qt63DLogic, qt6base-dev
+Qt63DQuick, qt6base-dev
+Qt63DQuickAnimation, qt6base-dev
+Qt63DQuickExtras, qt6base-dev
+Qt63DQuickInput, qt6base-dev
+Qt63DQuickRender, qt6base-dev
+Qt63DQuickScene2D, qt6base-dev
+Qt63DRender, qt6base-dev
+Qt6AssimpImporterPlugin, qt6base-dev
+Qt6AssimpSceneImportPlugin, qt6base-dev
+Qt6Bluetooth, qt6base-dev
+Qt6BluetoothTools, qt6base-dev
+Qt6BodymovinPrivate, qt6base-dev
+Qt6BuildInternals, qt6base-dev
+Qt6Charts, qt6base-dev
+Qt6ChartsQml, qt6base-dev
+Qt6Coap, qt6base-dev
+Qt6Concurrent, qt6base-dev
+Qt6Core, qt6base-dev
+Qt6Core5Compat, qt6base-dev
+Qt6CoreTools, qt6base-dev
+Qt6DBus, qt6base-dev
+Qt6DBusTools, qt6base-dev
+Qt6DataVisualization, qt6base-dev
+Qt6DataVisualizationQml2, qt6base-dev
+Qt6DefaultGeometryLoaderPlugin, qt6base-dev
+Qt6Designer, qt6base-dev
+Qt6DesignerComponentsPrivate, qt6base-dev
+Qt6DeviceDiscoverySupportPrivate, qt6base-dev
+Qt6DmaBufServerBufferIntegrationPlugin, qt6base-dev
+Qt6DmaBufServerBufferPlugin, qt6base-dev
+Qt6DrmEglServerBufferIntegrationPlugin, qt6base-dev
+Qt6DrmEglServerBufferPlugin, qt6base-dev
+Qt6EglFSDeviceIntegrationPrivate, qt6base-dev
+Qt6EglFsKmsGbmSupportPrivate, qt6base-dev
+Qt6EglFsKmsSupportPrivate, qt6base-dev
+Qt6FbSupportPrivate, qt6base-dev
+Qt6GLTFGeometryLoaderPlugin, qt6base-dev
+Qt6GLTFSceneExportPlugin, qt6base-dev
+Qt6GLTFSceneImportPlugin, qt6base-dev
+Qt6Gui, qt6base-dev
+Qt6GuiTools, qt6base-dev
+Qt6Help, qt6base-dev
+Qt6HostInfo, qt6base-dev
+Qt6HunspellInputMethodPrivate, qt6base-dev
+Qt6IIOSensorProxySensorPlugin, qt6base-dev
+Qt6InputSupportPrivate, qt6base-dev
+Qt6KmsSupportPrivate, qt6base-dev
+Qt6LabsAnimation, qt6base-dev
+Qt6LabsFolderListModel, qt6base-dev
+Qt6LabsQmlModels, qt6base-dev
+Qt6LabsSettings, qt6base-dev
+Qt6LabsSharedImage, qt6base-dev
+Qt6LabsWavefrontMesh, qt6base-dev
+Qt6Linguist, qt6base-dev
+Qt6LinguistTools, qt6base-dev
+Qt6Mqtt, qt6base-dev
+Qt6Multimedia, qt6base-dev
+Qt6MultimediaQuickPrivate, qt6base-dev
+Qt6MultimediaWidgets, qt6base-dev
+Qt6Network, qt6base-dev
+Qt6NetworkAuth, qt6base-dev
+Qt6Nfc, qt6base-dev
+Qt6OpcUa, qt6base-dev
+Qt6OpenGL, qt6base-dev
+Qt6OpenGLRendererPlugin, qt6base-dev
+Qt6OpenGLWidgets, qt6base-dev
+Qt6PacketProtocolPrivate, qt6base-dev
+Qt6PassThruCanBusPlugin, qt6base-dev
+Qt6PeakCanBusPlugin, qt6base-dev
+Qt6Positioning, qt6base-dev
+Qt6PositioningQuick, qt6base-dev
+Qt6PrintSupport, qt6base-dev
+Qt6QComposePlatformInputContextPlugin, qt6base-dev
+Qt6QCupsPrinterSupportPlugin, qt6base-dev
+Qt6QDebugMessageServiceFactoryPlugin, qt6base-dev
+Qt6QEglFSEmulatorIntegrationPlugin, qt6base-dev
+Qt6QEglFSIntegrationPlugin, qt6base-dev
+Qt6QEglFSKmsEglDeviceIntegrationPlugin, qt6base-dev
+Qt6QEglFSKmsGbmIntegrationPlugin, qt6base-dev
+Qt6QEglFSX11IntegrationPlugin, qt6base-dev
+Qt6QEvdevKeyboardPlugin, qt6base-dev
+Qt6QEvdevMousePlugin, qt6base-dev
+Qt6QEvdevTabletPlugin, qt6base-dev
+Qt6QEvdevTouchScreenPlugin, qt6base-dev
+Qt6QGeoPositionInfoSourceFactoryGeoclue2Plugin, qt6base-dev
+Qt6QGeoPositionInfoSourceFactoryNmeaPlugin, qt6base-dev
+Qt6QGeoPositionInfoSourceFactoryPollPlugin, qt6base-dev
+Qt6QGifPlugin, qt6base-dev
+Qt6QGtk3ThemePlugin, qt6base-dev
+Qt6QICNSPlugin, qt6base-dev
+Qt6QICOPlugin, qt6base-dev
+Qt6QIbusPlatformInputContextPlugin, qt6base-dev
+Qt6QJpegPlugin, qt6base-dev
+Qt6QLibInputPlugin, qt6base-dev
+Qt6QLinuxFbIntegrationPlugin, qt6base-dev
+Qt6QLocalClientConnectionFactoryPlugin, qt6base-dev
+Qt6QMYSQLDriverPlugin, qt6base-dev
+Qt6QMinimalEglIntegrationPlugin, qt6base-dev
+Qt6QMinimalIntegrationPlugin, qt6base-dev
+Qt6QNetworkManagerNetworkInformationPlugin, qt6base-dev
+Qt6QOffscreenIntegrationPlugin, qt6base-dev
+Qt6QOpen62541Plugin, qt6base-dev
+Qt6QPSQLDriverPlugin, qt6base-dev
+Qt6QQmlDebugServerFactoryPlugin, qt6base-dev
+Qt6QQmlDebuggerServiceFactoryPlugin, qt6base-dev
+Qt6QQmlInspectorServiceFactoryPlugin, qt6base-dev
+Qt6QQmlNativeDebugConnectorFactoryPlugin, qt6base-dev
+Qt6QQmlNativeDebugServiceFactoryPlugin, qt6base-dev
+Qt6QQmlPreviewServiceFactoryPlugin, qt6base-dev
+Qt6QQmlProfilerServiceFactoryPlugin, qt6base-dev
+Qt6QQuickProfilerAdapterFactoryPlugin, qt6base-dev
+Qt6QQuickWidgetPlugin, qt6base-dev
+Qt6QSQLiteDriverPlugin, qt6base-dev
+Qt6QScxmlEcmaScriptDataModelPlugin, qt6base-dev
+Qt6QSvgIconPlugin, qt6base-dev
+Qt6QSvgPlugin, qt6base-dev
+Qt6QTcpServerConnectionFactoryPlugin, qt6base-dev
+Qt6QTgaPlugin, qt6base-dev
+Qt6QTiffPlugin, qt6base-dev
+Qt6QTlsBackendCertOnlyPlugin, qt6base-dev
+Qt6QTlsBackendOpenSSLPlugin, qt6base-dev
+Qt6QTuioTouchPlugin, qt6base-dev
+Qt6QVirtualKeyboardPlugin, qt6base-dev
+Qt6QVkKhrDisplayIntegrationPlugin, qt6base-dev
+Qt6QVncIntegrationPlugin, qt6base-dev
+Qt6QWaylandBradientDecorationPlugin, qt6base-dev
+Qt6QWaylandDmabufClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandEglClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandEglClientBufferPlugin, qt6base-dev
+Qt6QWaylandEglPlatformIntegrationPlugin, qt6base-dev
+Qt6QWaylandEglStreamClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandFullScreenShellV1IntegrationPlugin, qt6base-dev
+Qt6QWaylandIntegrationPlugin, qt6base-dev
+Qt6QWaylandIviShellIntegrationPlugin, qt6base-dev
+Qt6QWaylandWlShellIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeEglClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeEglClientBufferPlugin, qt6base-dev
+Qt6QWaylandXCompositeEglPlatformIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeGlxClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeGlxClientBufferPlugin, qt6base-dev
+Qt6QWaylandXCompositeGlxPlatformIntegrationPlugin, qt6base-dev
+Qt6QWaylandXdgShellIntegrationPlugin, qt6base-dev
+Qt6QWbmpPlugin, qt6base-dev
+Qt6QWebpPlugin, qt6base-dev
+Qt6QXcbEglIntegrationPlugin, qt6base-dev
+Qt6QXcbGlxIntegrationPlugin, qt6base-dev
+Qt6QXcbIntegrationPlugin, qt6base-dev
+Qt6QXdgDesktopPortalThemePlugin, qt6base-dev
+Qt6Qml, qt6base-dev
+Qt6QmlCompilerPrivate, qt6base-dev
+Qt6QmlCore, qt6base-dev
+Qt6QmlDebugPrivate, qt6base-dev
+Qt6QmlDevToolsPrivate, qt6base-dev
+Qt6QmlDomPrivate, qt6base-dev
+Qt6QmlImportScanner, qt6base-dev
+Qt6QmlLocalStorage, qt6base-dev
+Qt6QmlModels, qt6base-dev
+Qt6QmlTools, qt6base-dev
+Qt6QmlWorkerScript, qt6base-dev
+Qt6QmlXmlListModel, qt6base-dev
+Qt6QtVirtualKeyboardHangulPlugin, qt6base-dev
+Qt6QtVirtualKeyboardHunspellPlugin, qt6base-dev
+Qt6QtVirtualKeyboardOpenWnnPlugin, qt6base-dev
+Qt6QtVirtualKeyboardPinyinPlugin, qt6base-dev
+Qt6QtVirtualKeyboardTCImePlugin, qt6base-dev
+Qt6QtVirtualKeyboardThaiPlugin, qt6base-dev
+Qt6Quick, qt6base-dev
+Qt6Quick3D, qt6base-dev
+Qt6Quick3DAssetImport, qt6base-dev
+Qt6Quick3DAssetUtils, qt6base-dev
+Qt6Quick3DEffects, qt6base-dev
+Qt6Quick3DHelpers, qt6base-dev
+Qt6Quick3DIblBaker, qt6base-dev
+Qt6Quick3DParticles, qt6base-dev
+Qt6Quick3DRuntimeRender, qt6base-dev
+Qt6Quick3DTools, qt6base-dev
+Qt6Quick3DUtils, qt6base-dev
+Qt6QuickControls2, qt6base-dev
+Qt6QuickControls2Impl, qt6base-dev
+Qt6QuickControlsTestUtilsPrivate, qt6base-dev
+Qt6QuickDialogs2, qt6base-dev
+Qt6QuickDialogs2QuickImpl, qt6base-dev
+Qt6QuickDialogs2Utils, qt6base-dev
+Qt6QuickLayouts, qt6base-dev
+Qt6QuickParticlesPrivate, qt6base-dev
+Qt6QuickShapesPrivate, qt6base-dev
+Qt6QuickTemplates2, qt6base-dev
+Qt6QuickTest, qt6base-dev
+Qt6QuickTestUtilsPrivate, qt6base-dev
+Qt6QuickTestplugin, qt6base-dev
+Qt6QuickTimeline, qt6base-dev
+Qt6QuickWidgets, qt6base-dev
+Qt6RemoteObjects, qt6base-dev
+Qt6RemoteObjectsQml, qt6base-dev
+Qt6RemoteObjectsTools, qt6base-dev
+Qt6RepParser, qt6base-dev
+Qt6Scene2DPlugin, qt6base-dev
+Qt6Scxml, qt6base-dev
+Qt6ScxmlQml, qt6base-dev
+Qt6ScxmlTools, qt6base-dev
+Qt6Sensors, qt6base-dev
+Qt6SensorsQuick, qt6base-dev
+Qt6SensorsQuickplugin, qt6base-dev
+Qt6SerialBus, qt6base-dev
+Qt6SerialBusTools, qt6base-dev
+Qt6SerialPort, qt6base-dev
+Qt6ShaderTools, qt6shadertools-dev
+Qt6ShaderToolsTools, qt6shadertools-dev
+Qt6ShmServerBufferIntegrationPlugin, qt6base-dev
+Qt6ShmServerBufferPlugin, qt6base-dev
+Qt6SocketCanBusPlugin, qt6base-dev
+Qt6Sql, qt6base-dev
+Qt6StateMachine, qt6base-dev
+Qt6StateMachineQml, qt6base-dev
+Qt6Svg, qt6base-dev
+Qt6SvgWidgets, qt6base-dev
+Qt6Test, qt6base-dev
+Qt6TinyCanBusPlugin, qt6base-dev
+Qt6Tools, qt6base-dev
+Qt6ToolsTools, qt6base-dev
+Qt6UiPlugin, qt6base-dev
+Qt6UiTools, qt6base-dev
+Qt6UipAssetImporterPlugin, qt6base-dev
+Qt6VirtualCanBusPlugin, qt6base-dev
+Qt6VirtualKeyboard, qt6base-dev
+Qt6VulkanServerBufferIntegrationPlugin, qt6base-dev
+Qt6VulkanServerBufferPlugin, qt6base-dev
+Qt6WaylandClient, qt6base-dev
+Qt6WaylandCompositor, qt6base-dev
+Qt6WaylandCompositorIviapplication, qt6base-dev
+Qt6WaylandCompositorWLShell, qt6base-dev
+Qt6WaylandCompositorXdgShell, qt6base-dev
+Qt6WaylandEglClientHwIntegrationPrivate, qt6base-dev
+Qt6WaylandEglCompositorHwIntegrationPrivate, qt6base-dev
+Qt6WaylandScannerTools, qt6base-dev
+Qt6WaylandTextureSharing, qt6base-dev
+Qt6WaylandTextureSharingExtension, qt6base-dev
+Qt6WebChannel, qt6base-dev
+Qt6WebSockets, qt6base-dev
+Qt6WebView, qt6base-dev
+Qt6WebViewQuick, qt6base-dev
+Qt6Widgets, qt6base-dev
+Qt6WidgetsTools, qt6base-dev
+Qt6WlShellIntegrationPrivate, qt6base-dev
+Qt6XcbQpaPrivate, qt6base-dev
+Qt6Xml, qt6base-dev
+Qt6declarative_opcua, qt6base-dev
+Qt6declarative_remoteobjects, qt6base-dev
+Qt6declarative_scxml, qt6base-dev
+Qt6dummySensorPlugin, qt6base-dev
+Qt6genericSensorPlugin, qt6base-dev
+Qt6labsanimationplugin, qt6base-dev
+Qt6labsmodelsplugin, qt6base-dev
+Qt6lottieqtplugin, qt6base-dev
+Qt6modelsplugin, qt6base-dev
+Qt6particlesplugin, qt6base-dev
+Qt6positioningquickplugin, qt6base-dev
+Qt6qmlfolderlistmodelplugin, qt6base-dev
+Qt6qmllocalstorageplugin, qt6base-dev
+Qt6qmlplugin, qt6base-dev
+Qt6qmlsettingsplugin, qt6base-dev
+Qt6qmlshapesplugin, qt6base-dev
+Qt6qmlwavefrontmeshplugin, qt6base-dev
+Qt6qmlwebsockets, qt6base-dev
+Qt6qmlxmllistmodelplugin, qt6base-dev
+Qt6qquick3dplugin, qt6base-dev
+Qt6qquicklayoutsplugin, qt6base-dev
+Qt6qtchartsqml2, qt6base-dev
+Qt6qtgraphicaleffectsplugin, qt6base-dev
+Qt6qtgraphicaleffectsprivate, qt6base-dev
+Qt6qtlabsplatformplugin, qt6base-dev
+Qt6qtqmlcoreplugin, qt6base-dev
+Qt6qtqmlstatemachine, qt6base-dev
+Qt6qtquick2plugin, qt6base-dev
+Qt6qtquick3dassetutilsplugin, qt6base-dev
+Qt6qtquick3deffectplugin, qt6base-dev
+Qt6qtquick3dhelpersplugin, qt6base-dev
+Qt6qtquick3dparticles3dplugin, qt6base-dev
+Qt6qtquickcontrols2basicstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2basicstyleplugin, qt6base-dev
+Qt6qtquickcontrols2fusionstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2fusionstyleplugin, qt6base-dev
+Qt6qtquickcontrols2imaginestyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2imaginestyleplugin, qt6base-dev
+Qt6qtquickcontrols2implplugin, qt6base-dev
+Qt6qtquickcontrols2materialstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2materialstyleplugin, qt6base-dev
+Qt6qtquickcontrols2nativestyleplugin, qt6base-dev
+Qt6qtquickcontrols2plugin, qt6base-dev
+Qt6qtquickcontrols2universalstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2universalstyleplugin, qt6base-dev
+Qt6qtquickdialogs2quickimplplugin, qt6base-dev
+Qt6qtquickdialogsplugin, qt6base-dev
+Qt6qtquickscene2dplugin, qt6base-dev
+Qt6qtquickscene3dplugin, qt6base-dev
+Qt6qtquicktemplates2plugin, qt6base-dev
+Qt6qtquicktimelineplugin, qt6base-dev
+Qt6qtquickvirtualkeyboardplugin, qt6base-dev
+Qt6qtquickvirtualkeyboardsettingsplugin, qt6base-dev
+Qt6qtquickvirtualkeyboardstylesplugin, qt6base-dev
+Qt6qtwebviewquickplugin, qt6base-dev
+Qt6quick3danimationplugin, qt6base-dev
+Qt6quick3dcoreplugin, qt6base-dev
+Qt6quick3dextrasplugin, qt6base-dev
+Qt6quick3dinputplugin, qt6base-dev
+Qt6quick3dlogicplugin, qt6base-dev
+Qt6quick3drenderplugin, qt6base-dev
+Qt6quickmultimedia, qt6base-dev
+Qt6quicktooling, qt6base-dev
+Qt6quickwindow, qt6base-dev
+Qt6qwaylandcompositorplugin, qt6base-dev
+Qt6sharedimageplugin, qt6base-dev
+Qt6webchannel, qt6base-dev
+Qt6workerscriptplugin, qt6base-dev
+QtBaseTests, qt6base-dev
+QtChartsTests, qt6base-dev
+QtCoapTests, qt6base-dev
+QtConnectivityTests, qt6base-dev
+QtDataVisualizationTests, qt6base-dev
+QtDeclarativeTests, qt6base-dev
+QtLocationTests, qt6base-dev
+QtLottieTests, qt6base-dev
+QtMqttTests, qt6base-dev
+QtMultimediaTests, qt6base-dev
+QtNetworkAuthTests, qt6base-dev
+QtOpcUaTests, qt6base-dev
+QtQuick3DTests, qt6base-dev
+QtQuickTimelineTests, qt6base-dev
+QtRemoteObjectsTests, qt6base-dev
+QtScxmlTests, qt6base-dev
+QtSensorsTests, qt6base-dev
+QtSerialBusTests, qt6base-dev
+QtSerialPortTests, qt6base-dev
+QtShaderToolsTests, qt6shadertools-dev
+QtSvgTests, qt6base-dev
+QtTesting, ParaView-dev
+QtToolsTests, qt6base-dev
+QtVirtualKeyboardTests, qt6base-dev
+QtWaylandScanner, extra-cmake-modules qtwayland-dev
+QtWaylandTests, qt6base-dev
+QtWebChannelTests, qt6base-dev
+QtWebSocketsTests, qt6base-dev
+QtWebViewTests, qt6base-dev
+QuaZip-Qt5, quazip-dev
+QuaZip5, quazip-dev
+RdKafka, librdkafka-dev
+Rinutils, rinutils-dev
+Ruby, ruby
+SDL, SDL-dev
+SDL_image, SDL_image-dev
+SDL_mixer, SDL_mixer-dev
+SDL_net, SDL_net-dev
+SDL_ttf, SDL_ttf-dev
+SFML, SFML-dev
+SPIRV-Headers, llvm-data
+SPIRV-Tools, SPIRV-Tools-dev
+SPIRV-Tools-link, SPIRV-Tools-dev
+SPIRV-Tools-lint, SPIRV-Tools-dev
+SPIRV-Tools-opt, SPIRV-Tools-dev
+SPIRV-Tools-reduce, SPIRV-Tools-dev
+SUNDIALS, sundials-dev
+SWIG, swig
+Sasl2, extra-cmake-modules pkgconfig(libsasl2)
+Savitar, libSavitar-dev
+ScreenSaverDBusInterface, kscreenlocker-dev
+Seccomp, extra-cmake-modules
+ShaderTools, qt6shadertools-dev
+SharedMimeInfo, extra-cmake-modules shared-mime-info
+Shiboken2, pyside2-setup-dev
+SignOnQt5, signond-dev
+Snappy, snappy-dev
+Subversion, subversion
+SymEngine, symengine-dev
+TCL, tcl-dev tk-dev
+TIFF, tiff-dev
+TclStub, tcl-dev
+Tclsh, tcl
+TelepathyQt5, telepathy-qt-dev
+TelepathyQt5Service, telepathy-qt-dev
+Threads, glibc-dev
+Torch, pypi(pytorch)
+UnixCommands, bash coreutils gzip
+UsePkg, cmake-data
+VPL, oneVPL-dev
+VPP, vpp-dev
+VTK, VTK-dev
+Vc, Vc-dev
+Volk, volk-dev
+Vulkan, Vulkan-Headers-dev Vulkan-Loader-dev Vulkan-Tools
+Waffle, waffle-dev
+Wayland, extra-cmake-modules pkgconfig(wayland-client)
+WaylandScanner, extra-cmake-modules wayland
+Wget, wget
+WinPR, FreeRDP-dev
+Wireshark, wireshark
+Wish, tk
+X11, libX11-dev libICE-dev libSM-dev libXau-dev libXcomposite-dev libXcursor-dev libXdamage-dev libXdmcp-dev libXext-dev libXfixes-dev libXft-dev libXi-dev libXinerama-dev libXi-dev libXmu-dev libXpm-dev libXrandr-dev libXrender-dev libXres-dev libXScrnSaver-dev libXt-dev libXtst-dev libXv-dev libXxf86vm-dev
+X11_XCB, extra-cmake-modules pkgconfig(x11-xcb)
+XCB, extra-cmake-modules pkgconfig(xcb) xcb-util-cursor-dev xcb-util-image-dev xcb-util-keysyms-dev xcb-util-renderutil-dev xcb-util-wm-dev xcb-util-dev
+Z3, Z3-dev
+ZLIB, zlib-dev
+ZXing, zxing-dev
+Zopfli, zopfli-dev
+ade, ade-data
+arpack-ng, arpack-ng-dev
+assimp, assimp-dev
+automotive-dlt, dlt-daemon-dev
+bash-completion, bash-completion-data
+box2d, box2d-dev
+cJSON, cJSON-dev
+catkin, catkin-data
+cereal, cereal-dev
+clFFT, clFFT-dev
+cmark, cmark-dev
+cmocka, cmocka-dev
+codec2, codec2-dev
+coin, Coin-dev
+cpprestsdk, cpprestsdk-dev
+cppzmq, cppzmq-data
+dbusmenu-qt5, libdbusmenu-qt-dev
+dlib, dlib-dev
+dnnl, mkl-dnn-dev
+double-conversion, double-conversion-dev
+eclipse-paho-mqtt-c, paho.mqtt.c-dev
+embree, embree-dev
+exiv2, exiv2-dev
+expat, expat-dev
+fann, fann-dev
+fm-qt, libfm-qt-data
+fmt, fmt-dev
+gflags, gflags-dev
+glfw3, glfw-dev
+glm, glm-dev
+glog, glog-dev
+glslang, glslang-data
+harfbuzz, harfbuzz-dev
+hiredis, hiredis-c-data
+igsc, igsc-dev
+json-c, json-c-dev
+kColorPicker, kcolorpicker-dev
+kImageAnnotator, kimageannotator-dev
+kf6.Config, kconfig-dev
+kf6.CoreAddons, kcoreaddons-dev
+kf6.Crash, kcrash-dev
+kf6.DBusAddons, kdbusaddons-dev
+kf6.FileMetaData, kfilemetadata-dev
+kf6.I18n, ki18n-dev
+kf6.JobWidgets, kjobwidgets-dev
+kf6.KIO, kio-dev
+kf6.Notifications, knotifications-dev
+kf6.Purpose, purpose-dev
+kf6.Runner, krunner-dev
+kf6.Service, kservice-dev
+kf6.StatusNotifierItem, kstatusnotifieritem-dev
+kim-api, kim-api-data
+leveldb, leveldb-dev
+libavif, libavif-dev
+libconfig++, libconfig-dev
+libconfig, libconfig-dev
+libcuckoo, libcuckoo-data
+libjpeg-turbo, libjpeg-turbo-dev
+libpala, palapeli
+libssh, libssh-dev
+libxml2, libxml2-dev
+libzip, libzip-dev
+lpcnetfreedv, LPCNet-dev
+lxqt, liblxqt-dev
+lxqt-build-tools, lxqt-build-tools-data
+lxqt-globalkeys, lxqt-globalkeys-dev
+lxqt-globalkeys-ui, lxqt-globalkeys-dev
+md4c, md4c-dev
+md4cHtml, md4c-dev
+msgpack, msgpack-c-dev
+nanoflann, nanoflann-dev
+paraview, ParaView-dev
+pugixml, pugixml-dev
+pybind11, pypi(pybind11)
+qt5xdg, libqtxdg-data
+qt5xdgiconloader, libqtxdg-data
+qt6.DBus, qt6base-dev
+qt6.Gui, qt6base-dev
+qt6.Widgets, qt6base-dev
+qtxdg-tools, qtxdg-tools-data
+rabbitmq-c, rabbitmq-c-dev
+realsense2, librealsense-dev
+realsense2-gl, librealsense-dev
+rttr, rttr-data
+sdl2, SDL2-dev
+sdl2_ttf, SDL2_ttf-dev
+spdlog, spdlog-dev
+spirv_cross_c, SPIRV-Cross-data
+spirv_cross_c_shared, SPIRV-Cross-data
+spirv_cross_core, SPIRV-Cross-data
+spirv_cross_cpp, SPIRV-Cross-data
+spirv_cross_glsl, SPIRV-Cross-data
+spirv_cross_hlsl, SPIRV-Cross-data
+spirv_cross_msl, SPIRV-Cross-data
+spirv_cross_reflect, SPIRV-Cross-data
+spirv_cross_util, SPIRV-Cross-data
+sysstat-qt5, libsysstat-data
+tinyxml2, tinyxml2-dev
+tsl-robin-map, robin-map-data
+utf8cpp, utf8cpp-dev
+uwac, FreeRDP-dev
+vtk, ParaView-dev
+vtk, VTK-dev
+x86-64, cmake
+xapian, xapian-core-dev
+yaml-cpp, yaml-cpp-dev

autospec/commitmessage.py

@@ -22,149 +22,279 @@
 #
 #
 
+import os
 import re
+import shutil
+import sys
+from subprocess import PIPE, run
 
-import build
-import config
-import tarball
-
-commitmessage = []
-
-cves = set()
-have_cves = False
-cvestring = ""
+import util
 
 
-def new_cve(cve):
-    global cvestring
-    global have_cves
-    global cves
+def scan_for_changes(download_path, directory, transforms):
+    """Scan for changelogs or news files in the file sources.
 
-    cves.add(cve)
-    have_cves = True
-    cvestring += " " + cve
+    Scan for changelogs or news files in the source code and copy them to download_path as their
+    `transform`ed name. The file with the transformed name will later be parsed to find the
+    commit message.
+    """
+    found = []
+    interests = transforms.keys()
+    for dirpath, dirnames, files in os.walk(directory, topdown=False):
+        hits = [x for x in files if x.lower() in interests and x.lower() not in found]
+        for item in hits:
+            source = os.path.join(dirpath, item)
+            target = os.path.join(download_path, transforms[item.lower()])
+            try:
+                shutil.copy(source, target)
+                os.chmod(target, 0o644)
+            except Exception as e:
+                util.print_fatal("Error copying file: {}".format(e))
+                sys.exit(1)
+            found.append(item)
 
 
-def process_NEWS(file):
-    global commitmessage
-    news = []
-    start = 0
-    stop = 0
-    success = 0
+def is_header(lines, curindex):
+    """Check if the current line is a section header.
 
-    if config.old_version is None or config.old_version == tarball.version:
-        return
+    Check if the current line is a section header by looking for a blank line before it or an
+    underline/section break (---) after it. Returns True for lines at the beginning or end of the
+    file.
+    """
+    if curindex == 0:
+        # treat the start of the file as a header
+        # this will not be caught by an IndexError because -1 is a valid index
+        return True
 
     try:
-        with open(build.download_path + "/" + file) as f:
-            news = f.readlines()
+        return (not lines[curindex - 1]) or ('---' in lines[curindex + 1])
+    except IndexError:
+        # end of file doesn't matter for starting a block and is an obvious end
+        # of a block
+        return True
+
+
+def find_in_line(pattern, line):
+    """Return True if the pattern is in the line, False otherwise."""
+    return bool(re.search(pattern, line))
+
+
+def process_NEWS(newsfile, old_version, name, version, download_path):
+    """Parse the newfile for relevent changes.
+
+    Look for changes and CVE fixes relevant to current version update. This information is returned
+    as a tuple: (commitmessage, cves).
+
+    A maximum of 15 lines from the newsfile is returned in the commitmessage.
+    If the newsfile information is truncated to 15 lines an additional line is
+    added "(NEWS truncated at 15 lines)"
+    """
+    commitmessage = []
+    cves = set()
+    start = 0
+    stop = 0
+    success = False
+    start_found = False
+
+    if old_version is None or old_version == version:
+        # no version update, so no information to search for in newsfile
+        return commitmessage, cves
+
+    try:
+        with util.open_auto(os.path.join(download_path, newsfile)) as f:
+            newslines = f.readlines()
     except EnvironmentError:
-        return
+        return commitmessage, cves
 
-    stop = len(news)
-    if stop <= start:
-        return
+    newslines = [news.rstrip('\n') for news in newslines]
 
-    i = start
-    while i < stop:
-        news[i] = news[i].strip('\n')
-        i = i + 1
+    # escape some values for use in regular expressions below
+    escaped_curver = re.escape(version)
+    escaped_oldver = re.escape(old_version)
+    escaped_tarname = re.escape(name)
 
-    i = start + 1
-    while i < stop - 1:
-        if news[i] == config.old_version and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == "Overview of changes leading to " + config.old_version and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == config.old_version + ":" and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == tarball.version + ":" and news[i - 1] == "":
-            start = i
-        if news[i] == tarball.name + "-" + config.old_version + ":" and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == tarball.name + "-" + tarball.version + ":" and news[i - 1] == "":
-            start = i
-        if news[i] == "- " + config.old_version + ":" and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == "- " + tarball.version + ":" and news[i - 1] == "":
-            start = i
-        if news[i].find(config.old_version) >= 0 and news[i].find("*** Changes in ") >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].find(config.old_version) >= 0 and news[i].find("201") >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].lower().find(tarball.name + " " + config.old_version) >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].find(config.old_version) >= 0 and news[i].find("Version ") >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].find(tarball.version) >= 0 and news[i].find("Version ") >= 0 and news[i - 1] == "":
-            start = i
+    # these are patterns that define the beginning of a block of information
+    # regarding the current version.
+    news_start = [r'Version.*{}'.format(escaped_curver),
+                  r'(v|- )?{}:?'.format(escaped_curver),
+                  r'{}-{}:?'.format(escaped_tarname, escaped_curver),
+                  r'{} 20'.format(escaped_curver)]
 
-        if news[i].find(config.old_version + ":") == 0:
-            stop = i - 1
-            success = 1
-        if news[i] == config.old_version and news[i + 1].find("---") >= 0:
-            stop = i - 1
-            success = 1
-        if news[i] == tarball.version and news[i + 1].find("---") >= 0:
-            start = i
+    # these are patterns that define the end of a block of information
+    # regarding the current version.
+    news_end = [r'\*\*\* Changes in.*{}'.format(escaped_oldver),
+                r'{}.*201'.format(escaped_oldver),
+                r'Version.*{}'.format(escaped_oldver),
+                r'^Overview of changes leading to {}'.format(escaped_oldver),
+                r'^{}(-| ){}:?'.format(escaped_tarname, escaped_oldver),
+                r'v?{}:?'.format(escaped_oldver)]
 
-        i = i + 1
+    for idx, news in enumerate(newslines):
+        # only check headers for begin and end patterns
+        if is_header(newslines, idx):
+            for pat in news_start:
+                if find_in_line(pat, news):
+                    start = idx
+                    start_found = True
+                    break
+            if start_found:
+                for pat in news_end:
+                    if find_in_line(pat, news):
+                        success = True
+                        stop = idx - 1  # stop before this header
+                        break
+            if start_found and success:
+                break
 
-    if success == 0:
-        return
+    if not success or stop <= start:
+        return commitmessage, cves
 
     # now search for CVEs
-    i = start
-    pat = re.compile("(CVE\-[0-9]+\-[0-9]+)")
-    while i < stop and i < start:
-        match = pat.search(news[i])
+    pat = re.compile(r"(CVE\-[0-9]+\-[0-9]+)")
+    for news in newslines[start:stop]:
+        match = pat.search(news)
         if match:
-                s = match.group(1)
-                new_cve(s)
-        i = i + 1
+            s = match.group(1)
+            cves.add(s)
+
+    # compile commitmessage to return
+    commitmessage.append("")
+    for news in newslines[start:min(start + 15, stop)]:
+        commitmessage.append(news)
+
+    if stop > start + 15:
+        # append message that news was truncated
+        commitmessage.extend(["", "(NEWS truncated at 15 lines)"])
 
     commitmessage.append("")
-    i = start
-    while i < stop and i < start + 15:
-        commitmessage.append(news[i])
-        i = i + 1
-    commitmessage.append("")
+    return commitmessage, cves
 
 
-def guess_commit_message():
-    global cvestring
+def process_git(giturl, oldversion, newversion, name):
+    """Check out a git tree and try to turn the git history into a commit message."""
+    oldtag = ""
+    guessed_oldtag = oldversion
+    newtag = ""
+    guessed_newtag = newversion
+
+    if len(giturl) < 1:
+        return ""
+    if oldversion == newversion:
+        return ""
+
+    run(["git", "-C", "results", "clone", giturl, name])
+    p = run(["git", "-C", "results/" + name, "tag"], stdout=PIPE)
+    tags = p.stdout.decode('utf-8').split('\n')
+
+    # Version strings will sometimes have a leading 'v', 'V', or
+    # '<packagename>-' prefix, or possibly a combination of these.
+    regex_template = r'^({}-)?[vV]?{{}}$'.format(re.escape(name))
+
+    for t in tags:
+        i = t.find(oldversion)
+        if i != -1:
+            guessed_oldtag = t
+        pat = regex_template.format(re.escape(oldversion))
+        if re.search(pat, t):
+            oldtag = t
+        i = t.find(newversion)
+        if i != -1:
+            guessed_newtag = t
+        pat = regex_template.format(re.escape(newversion))
+        if re.search(pat, t):
+            newtag = t
+
+    if oldtag == "":
+        oldtag = guessed_oldtag
+    if newtag == "":
+        newtag = guessed_newtag
+
+    p = run(["git", "-C", "results/" + name, "log", "--no-merges", oldtag + ".." + newtag], stdout=PIPE)
+    fulllog = p.stdout.decode('utf-8', errors='replace').split('\n')
+    # 'git shortlog' can accept any 'git log' output over stdin, so make sure
+    # it lacks merge commits, too.
+    p = run(["git", "-C", "results/" + name, "shortlog"], input=p.stdout, stdout=PIPE)
+    shortlog = p.stdout.decode('utf-8', errors='replace').split('\n')
+
+    if len(fulllog) < 15:
+        return fulllog
+    else:
+        return shortlog
+
+
+def guess_commit_message(keyinfo, config, content):
+    """Parse newsfile for a commit message.
+
+    Try and find a sane commit message for the newsfile. The commit message defaults to the
+    following for an updated version if no CVEs are fixed:
+
+    <tarball name>: Autospec creation for update from version <old> to version <new>
+
+    If CVEs are fixed:
+
+    <tarball name>: Fix for <cve>
+
+    And if the version does not change:
+
+    <tarball name>: Autospec creation for version <version>
+
+    Additional information is appended to the commitmessage depending on NEWS
+    and ChangeLog files and the presence of CVEs. The commitmessage is written
+    to a file at <download path>/commitmsg.
+    """
+    cvestring = ""
+    cves = set()
+    commitmessage = []
+    for cve in config.cves:
+        cves.add(cve)
+        cvestring += " " + cve
 
     # default commit messages before we get too smart
-    if config.old_version is not None and config.old_version != tarball.version:
-        commitmessage.append(tarball.name + ": Autospec creation for update from version " +
-                             config.old_version + " to version " +
-                             tarball.version)
+    if config.old_version is not None and config.old_version != content.version:
+        commitmessage.append("{}: Autospec creation for update from version {} to version {}"
+                             .format(content.name, config.old_version, content.version))
+        if content.giturl != "":
+            gitmsg = process_git(content.giturl, config.old_version, content.version, content.name)
+            commitmessage.append("")
+            commitmessage.extend(gitmsg)
     else:
-        if have_cves:
-            commitmessage.append(tarball.name + ": Fix for " + cvestring.strip())
+        if cves:
+            commitmessage.append("{}: Fix for {}"
+                                 .format(content.name, cvestring.strip()))
         else:
-            commitmessage.append(tarball.name + ": Autospec creation for version " +
-                                 tarball.version)
+            commitmessage.append("{}: Autospec creation for version {}"
+                                 .format(content.name, content.version))
     commitmessage.append("")
 
-    if have_cves:
+    # Only use Changelog if the giturl isn't defined as it is often
+    # duplicate content from the git log.
+    if content.giturl:
+        newsfiles = ["NEWS"]
+    else:
+        newsfiles = ["NEWS", "ChangeLog"]
+    for newsfile in newsfiles:
+        # parse news files for relevant version updates and cve fixes
+        newcommitmessage, newcves = process_NEWS(newsfile, config.old_version, content.name, content.version, config.download_path)
+        commitmessage.extend(newcommitmessage)
+        cves.update(newcves)
+
+    if cves:
+        # make the package security sensitive if a CVE was patched
+        config.config_opts['security_sensitive'] = True
+        config.rewrite_config_opts()
+        # append CVE fixes to end of commit message
         commitmessage.append("CVEs fixed in this build:")
-        commitmessage.extend(list(cves))
+        commitmessage.extend(sorted(list(cves)))
         commitmessage.append("")
 
-    process_NEWS("NEWS")
-    process_NEWS("ChangeLog")
+    if keyinfo:
+        commitmessage.append("Key imported:\n{}".format(keyinfo))
+
+    util.write_out(os.path.join(config.download_path, "commitmsg"),
+                   "\n".join(commitmessage) + "\n")
 
     print("Guessed commit message:")
-    print(commitmessage)
-    with open(build.download_path + "/commitmsg", "w") as file:
-        file.write("\n".join(commitmessage) + "\n")
+    try:
+        print("\n".join(commitmessage))
+    except Exception:
+        print("Can't print")

autospec/configure_blacklist

@@ -0,0 +1 @@
+whether double complex BLAS can be used

autospec/configure_whitelist

@@ -0,0 +1,680 @@
+acroread
+acroread4
+af77
+a french Unicode locale
+altivec.h
+altivec.h presence
+altivec.h usability
+ANSI C header files
+atan2pi
+atanpi
+a traditional french locale
+a traditional japanese locale
+a transitional chinese locale
+bison
+broken visibility attribute
+BSDgettimeofday
+/builddir/build/BUILD/gcc-build/./gcc/xgcc -B/builddir/build/BUILD/gcc-build/./gcc/ -B/usr/x86_64-generic-linux/bin/ -B/usr/x86_64-generic-linux/lib/ -isystem /usr/x86_64-generic-linux/include -isystem /usr/x86_64-generic-linux/sys-include    option to accept ISO C89
+c_asm.h
+c_asm.h presence
+c_asm.h usability
+cf77
+CFLocaleCopyCurrent
+CFPreferencesCopyAppValue
+com_err.h
+Configure miss: struct stat.st_atimespec
+cospi
+cospi in -lsunmath
+crt_externs.h
+crt_externs.h presence
+crt_externs.h usability
+dependency style of g++
+dependency style of gcc
+direct.h
+dl.h
+dl.h presence
+dl.h usability
+dlltool
+dlopen
+_doprnt
+doxygen
+dummy main to link with Fortran 77 libraries
+dynamic linker characteristics
+efc
+--enable-generated-files-in-srcdir
+--enable-version-specific-runtime-libs
+--enable-vtable-verify
+epcf90
+etags
+et/com_err.h
+evince
+external symbol _system_configuration
+extra flags to get ANSI library prototypes
+f77
+f90
+f95
+fdwalk
+_FILE_OFFSET_BITS value needed for large files
+firefox
+fl32
+flex
+floatingpoint.h
+floatingpoint.h presence
+floatingpoint.h usability
+fop
+fort
+fort77
+Fortran flag to compile .f90 files
+Fortran flag to compile .f95 files
+fprintf_unlocked
+frt
+g77
+galeon
+gccgo
+gcc option to accept ANSI C
+gcc option to accept ISO C89
+gcc option to accept ISO C99
+gcc option to enable C11 features
+gcj
+gethrtime
+getpass_r
+getsysinfo
+ggv
+gm4
+gnatbind
+gnatmake
+gnome-gv
+gnome-moz-remote
+gnum4
+gnutar
+gobject-introspection
+gpdf
+gtar
+gv
+how to run the C preprocessor
+HP-UX
+hrtime_t
+i686-pc-linux-gnu-ar
+i686-pc-linux-gnu-dlltool
+i686-pc-linux-gnu-gcc option to accept ISO C89
+i686-pc-linux-gnu-mt
+i686-pc-linux-gnu-objdump
+i686-pc-linux-gnu-ranlib
+i686-pc-linux-gnu-strip
+iconv in -liconv
+iconvlist
+ieeefp.h
+ieeefp.h presence
+ieeefp.h usability
+if /builddir/build/BUILD/gcc-build/./gcc/xgcc -B/builddir/build/BUILD/gcc-build/./gcc/ -B/usr/x86_64-generic-linux/bin/ -B/usr/x86_64-generic-linux/lib/ -isystem /usr/x86_64-generic-linux/include -isystem /usr/x86_64-generic-linux/sys-include    static flag -static works
+if /builddir/build/BUILD/gcc-build/./gcc/xgcc -B/builddir/build/BUILD/gcc-build/./gcc/ -B/usr/x86_64-generic-linux/bin/ -B/usr/x86_64-generic-linux/lib/ -isystem /usr/x86_64-generic-linux/include -isystem /usr/x86_64-generic-linux/sys-include    supports -fno-rtti -fno-exceptions
+ifc
+if closesocket can be linked
+if CloseSocket can be linked
+if closesocket might be used
+if CloseSocket might be used
+if compiler is clang
+if compiler is DEC/Compaq/HP C
+if compiler is HP-UX C
+if compiler is IBM C
+if compiler is Intel C
+if compiler is LCC
+if compiler is SGI MIPS C
+if compiler is SGI MIPSpro C
+if compiler is SunPro C
+if compiler is Tiny C
+if compiler is Watcom C
+if compiler supports -Werror=unknown-warning-option
+if compiler supports -Werror=unused-command-line-argument
+if c++ static flag -static works
+if ctime_r wants three arguments
+if f95 static flag -static works
+if gcc static flag -static works
+if gcc -std=gnu99 supports -fno-rtti -fno-exceptions
+if gcc supports -fno-rtti -fno-exceptions
+if gcc supports -Werror=unknown-warning-option
+if gcc supports -Werror=unused-command-line-argument
+if gethostbyaddr_r takes 5 args.
+if gethostbyaddr_r takes 7 args.
+if gethostbyaddr_r wants seven arguments
+if gethostbyname_r takes 3 args.
+if gethostbyname_r takes 5 args.
+if gethostbyname_r wants five arguments
+if getservbyport_r takes 4 args.
+if getservbyport_r takes 5 args.
+if gfortran PIC flag -fPIC works
+if gfortran static flag -static works
+if gfortran supports -c -o file.o
+if g++ static flag -static works
+if inet_ntoa_r can be linked
+if inet_ntoa_r might be used
+if ioctlsocket can be linked
+if IoctlSocket can be linked
+if ioctlsocket might be used
+if IoctlSocket might be used
+if : is a manifest tool
+if libcurl version >= 7.28.0
+if memrchr is prototyped
+if memrchr might be used
+if mkdir takes one argument
+if more special flags are required for pthreads
+if setsockopt SO_NONBLOCK is compilable
+if setsockopt SO_NONBLOCK might be used
+if sig_atomic_t is already defined as volatile
+if sigsetjmp can be linked
+if strcmpi can be linked
+if strcmpi might be used
+if strerror_r is glibc like
+if stricmp can be linked
+if stricmp might be used
+if strncmpi can be linked
+if strncmpi might be used
+if strnicmp can be linked
+if strnicmp might be used
+if /usr/bin/gcc static flag -static works
+if /usr/bin/gcc -std=gnu99 static flag -static works
+if /usr/bin/gcc -std=gnu99 supports -fno-rtti -fno-exceptions
+if /usr/bin/gcc supports -fno-rtti -fno-exceptions
+if /usr/local/bin/gcc supports -fno-rtti -fno-exceptions
+if <X11/extensions/XIproto.h> is needed for xReply
+intrinsics.h
+intrinsics.h presence
+intrinsics.h usability
+io.h
+io.h presence
+io.h usability
+is_selinux_enabled in -lselinux
+issetugid
+Kerberos 5
+kfmclient
+kghostview
+kpdf
+kpsewhich
+kqueue
+krb5-config
+_LARGEFILE_SOURCE value needed for large files
+lchmod
+ldap_ntlm_bind
+lex library
+lf95
+libpapi... checking for papiServiceCreate in -lpapi
+library containing bindtextdomain
+library containing clock_gettime
+library containing connect
+library containing getaddrinfo
+library containing gethostent
+library containing setsockopt
+library containing socket
+library containing strerror
+libutil.h
+libutil.h presence
+libutil.h usability
+lipo
+lp
+lpr
+mach_absolute_time
+machine/hal_sysinfo.h
+mach/mach_time.h
+mach/mach_time.h presence
+mach/mach_time.h usability
+main in -linet
+makeindex
+Minix Amsterdam compiler
+minix/config.h
+minix/config.h presence
+minix/config.h usability
+minix/sha2.h
+minix/sha2.h presence
+minix/sha2.h usability
+missing gethostname prototype
+_mm_free
+_mm_malloc
+mozilla
+mt
+multithread API to use
+nanotime
+native Win32
+net/errno.h
+net/errno.h presence
+net/errno.h usability
+_NSGetEnviron
+ocamlbuild
+okular
+open
+OpenLDAP
+opera
+pcre/pcre.h
+pcre/pcre.h presence
+pcre/pcre.h usability
+pdflatex
+pdftex
+perl5
+pgf77
+pgf90
+pgf95
+pghpf
+pown
+process.h
+pstat_getdynamic
+pthread_create
+read_real_time
+_rtc intrinsic
+__secure_getenv
+sepol_check_context in -lsepol
+__setfpucw
+_set_invalid_parameter_handler
+setmode
+setproctitle
+shl_load
+shl_load in -ldld
+sin
+sinpi
+_snprintf
+socket in -lsocket
+some Win32 platform
+spawnve
+special C compiler options needed for large files
+stricmp
+strlcat
+strlcpy
+strnicmp
+struct sockaddr_in.sin_len
+struct sockaddr.sa_len
+struct sockaddr_un.sun_len
+struct stat.st_birthtimensec
+struct stat.st_birthtimespec.tv_nsec
+struct stat.st_birthtim.tv_nsec
+struct tm.__tm_gmtoff
+SunLDAP
+sunmath.h
+sunmath.h presence
+sunmath.h usability
+sys/endian.h
+sys/endian.h presence
+sys/endian.h usability
+sys/filio.h
+sys/filio.h presence
+sys/filio.h usability
+sys/limits.h
+sys/loadavg.h
+sys/loadavg.h presence
+sys/loadavg.h usability
+sys/mkdev.h
+sys/mkdev.h presence
+sys/mkdev.h usability
+sys/pstat.h
+sysroot
+sys/sockio.h
+sys/sockio.h presence
+sys/sockio.h usability
+sys/systeminfo.h
+sys/systeminfo.h presence
+sys/systeminfo.h usability
+table
+tanpi
+tex
+texi2any
+texi2dvi
+the BeOS
+the pthreads library -llthread
+the pthreads library -lpthreads
+time_base_to_time
+/usr/bin/gcc option to accept ISO C89
+/usr/local/bin/gcc option to accept ISO C89
+util.h
+util.h presence
+util.h usability
+valgrind.h
+valgrind.h presence
+vasnprintf
+vfork.h
+vfork.h presence
+vfork.h usability
+_vsnprintf
+whether alloca needs Cray hooks
+whether byte ordering is bigendian
+whether bzip2 support needs to be compiled
+whether  can compile ObjC++
+whether canonicalize_file_name must be declared
+whether char is unsigned
+whether __clang__ is declared
+whether closedir returns void
+whether compiler driver understands Ada
+whether cosq is declared
+whether C runtime needs -D__NO_MATH_INLINES
+whether dirfd is a macro
+whether f95 appends extra underscores to external names
+whether fprintf_unlocked is declared
+whether fpurge is declared without a macro
+whether g++ can compile ObjC++
+whether gcc accepts -g
+whether GCC assembler is compatible for ARM assembly implementations
+whether gcc needs -traditional
+whether gets is declared without a macro
+whether gettimeofday clobbers localtime buffer
+whether g++  supports C++11 features by default
+whether included gettext is requested
+whether integer division by zero raises SIGFPE
+whether __INTEL_COMPILER is declared
+whether // is distinct from /
+whether KERN_USRSTACK sysctl is supported
+whether -lc should be explicitly linked in
+whether long double and double are the same
+whether mktime sets errno
+whether mktime works correctly outside 1902-2037
+whether PCRE support needs to be compiled
+whether pthreads work with -kthread
+whether pthreads work with -Kthread
+whether pthreads work without any flags
+whether putenv("FOO=") can unset an environment variable
+whether sinq is declared
+whether _snprintf is declared
+whether _snwprintf is declared
+whether stat accepts an empty string
+whether stat file-mode macros are broken
+whether stricmp is declared
+whether strlcpy is declared
+whether strnicmp is declared
+whether __SUNPRO_C is declared
+whether system header files limit the line length
+whether termios.h defines TIOCGWINSZ
+whether the compiler supports -xldscope=hidden
+whether the inttypes.h PRIxNN macros are broken
+whether the thread-local storage support is from emutls
+whether to build gtk-doc documentation
+whether to build shared libraries
+whether to build static libraries
+whether to enable maintainer-specific portions of Makefiles
+whether to install libiberty headers and static library
+whether to use ARM IWMMXT intrinsics
+whether to use ARM NEON assembler
+whether to use ARM SIMD assembler
+whether to use Loongson MMI assembler
+whether to use MIPS DSPr2 assembler
+whether to use VMX/Altivec intrinsics
+Whether to write dependencies into .pc files
+whether we are cross compiling
+whether we are using the GNU Objective C compiler
+whether we can compute ObjC Make dependencies
+whether _XOPEN_SOURCE should be defined
+whether yytext is a pointer
+whether zlib support needs to be compiled
+Win32
+Win32 threads
+windows.h
+windows.h presence
+windows.h usability
+x86_64-generic-linux-ar
+x86_64-generic-linux-as
+x86_64-generic-linux-cc
+x86_64-generic-linux-dlltool
+x86_64-generic-linux-gccgo
+x86_64-generic-linux-gcj
+x86_64-generic-linux-gnu-aCC
+x86_64-generic-linux-gnu-af77
+x86_64-generic-linux-gnu-ar
+x86_64-generic-linux-gnu-c++
+x86_64-generic-linux-gnu-cc
+x86_64-generic-linux-gnu-cc++
+x86_64-generic-linux-gnu-CC
+x86_64-generic-linux-gnu-cf77
+x86_64-generic-linux-gnu-cl
+x86_64-generic-linux-gnu-cl.exe
+x86_64-generic-linux-gnu-cxx
+x86_64-generic-linux-gnu-dlltool
+x86_64-generic-linux-gnu-efc
+x86_64-generic-linux-gnu-epcf90
+x86_64-generic-linux-gnu-f77
+x86_64-generic-linux-gnu-f90
+x86_64-generic-linux-gnu-f95
+x86_64-generic-linux-gnu-FCC
+x86_64-generic-linux-gnu-fl32
+x86_64-generic-linux-gnu-fort
+x86_64-generic-linux-gnu-fort77
+x86_64-generic-linux-gnu-frt
+x86_64-generic-linux-gnu-ftn
+x86_64-generic-linux-gnu-g++
+x86_64-generic-linux-gnu-g77
+x86_64-generic-linux-gnu-g95
+x86_64-generic-linux-gnu-gcc
+x86_64-generic-linux-gnu-gfortran
+x86_64-generic-linux-gnu-gpp
+x86_64-generic-linux-gnu-ifc
+x86_64-generic-linux-gnu-ifort
+x86_64-generic-linux-gnu-KCC
+x86_64-generic-linux-gnu-lf95
+x86_64-generic-linux-gnu-lib
+x86_64-generic-linux-gnu-link
+x86_64-generic-linux-gnu-mt
+x86_64-generic-linux-gnu-nagfor
+x86_64-generic-linux-gnu-nm
+x86_64-generic-linux-gnu-objc
+x86_64-generic-linux-gnu-objcc
+x86_64-generic-linux-gnu-objdump
+x86_64-generic-linux-gnu-pgf77
+x86_64-generic-linux-gnu-pgf90
+x86_64-generic-linux-gnu-pgf95
+x86_64-generic-linux-gnu-pgfortran
+x86_64-generic-linux-gnu-pghpf
+x86_64-generic-linux-gnu-ranlib
+x86_64-generic-linux-gnu-RCC
+x86_64-generic-linux-gnu-strings
+x86_64-generic-linux-gnu-strip
+x86_64-generic-linux-gnu-windres
+x86_64-generic-linux-gnu-xlC
+x86_64-generic-linux-gnu-xlC_r
+x86_64-generic-linux-gnu-xlf
+x86_64-generic-linux-gnu-xlf90
+x86_64-generic-linux-gnu-xlf95
+x86_64-generic-linux-ld
+x86_64-generic-linux-lipo
+x86_64-generic-linux-nm
+x86_64-generic-linux-objcopy
+x86_64-generic-linux-objdump
+x86_64-generic-linux-ranlib
+x86_64-generic-linux-readelf
+x86_64-generic-linux-strip
+x86_64-generic-linux-windmc
+x86_64-generic-linux-windres
+x86_64-koji-linux-gnu-aCC
+x86_64-koji-linux-gnu-af77
+x86_64-koji-linux-gnu-ar
+x86_64-koji-linux-gnu-c++
+x86_64-koji-linux-gnu-cc++
+x86_64-koji-linux-gnu-CC
+x86_64-koji-linux-gnu-cl
+x86_64-koji-linux-gnu-cxx
+x86_64-koji-linux-gnu-dlltool
+x86_64-koji-linux-gnu-efc
+x86_64-koji-linux-gnu-epcf90
+x86_64-koji-linux-gnu-f77
+x86_64-koji-linux-gnu-f90
+x86_64-koji-linux-gnu-f95
+x86_64-koji-linux-gnu-FCC
+x86_64-koji-linux-gnu-fl32
+x86_64-koji-linux-gnu-fort
+x86_64-koji-linux-gnu-fort77
+x86_64-koji-linux-gnu-frt
+x86_64-koji-linux-gnu-g++
+x86_64-koji-linux-gnu-g77
+x86_64-koji-linux-gnu-gfortran
+x86_64-koji-linux-gnu-gpp
+x86_64-koji-linux-gnu-ifc
+x86_64-koji-linux-gnu-KCC
+x86_64-koji-linux-gnu-lf95
+x86_64-koji-linux-gnu-mt
+x86_64-koji-linux-gnu-objdump
+x86_64-koji-linux-gnu-pgf77
+x86_64-koji-linux-gnu-pgf90
+x86_64-koji-linux-gnu-pgf95
+x86_64-koji-linux-gnu-ranlib
+x86_64-koji-linux-gnu-RCC
+x86_64-koji-linux-gnu-strings
+x86_64-koji-linux-gnu-strip
+x86_64-koji-linux-gnu-windres
+x86_64-koji-linux-gnu-xlC
+x86_64-koji-linux-gnu-xlC_r
+x86_64-koji-linux-gnu-xlf
+x86_64-koji-linux-gnu-xlf90
+x86_64-koji-linux-gnu-xlf95
+x86_64-pc-linux-gnu-aCC
+x86_64-pc-linux-gnu-af77
+x86_64-pc-linux-gnu-ar
+x86_64-pc-linux-gnu-c++
+x86_64-pc-linux-gnu-cc++
+x86_64-pc-linux-gnu-CC
+x86_64-pc-linux-gnu-cl
+x86_64-pc-linux-gnu-cl.exe
+x86_64-pc-linux-gnu-cxx
+x86_64-pc-linux-gnu-dlltool
+x86_64-pc-linux-gnu-efc
+x86_64-pc-linux-gnu-epcf90
+x86_64-pc-linux-gnu-f77
+x86_64-pc-linux-gnu-f90
+x86_64-pc-linux-gnu-f95
+x86_64-pc-linux-gnu-FCC
+x86_64-pc-linux-gnu-fl32
+x86_64-pc-linux-gnu-fort
+x86_64-pc-linux-gnu-fort77
+x86_64-pc-linux-gnu-frt
+x86_64-pc-linux-gnu-g++
+x86_64-pc-linux-gnu-g77
+x86_64-pc-linux-gnu-gcc
+x86_64-pc-linux-gnu-gfortran
+x86_64-pc-linux-gnu-gpp
+x86_64-pc-linux-gnu-ifc
+x86_64-pc-linux-gnu-KCC
+x86_64-pc-linux-gnu-lf95
+x86_64-pc-linux-gnu-mt
+x86_64-pc-linux-gnu-objdump
+x86_64-pc-linux-gnu-pgf77
+x86_64-pc-linux-gnu-pgf90
+x86_64-pc-linux-gnu-pgf95
+x86_64-pc-linux-gnu-pkg-config
+x86_64-pc-linux-gnu-ranlib
+x86_64-pc-linux-gnu-RCC
+x86_64-pc-linux-gnu-strip
+x86_64-pc-linux-gnu-xlC
+x86_64-pc-linux-gnu-xlC_r
+x86_64-pc-linux-gnu-xlf
+x86_64-pc-linux-gnu-xlf90
+x86_64-pc-linux-gnu-xlf95
+x86_64-unknown-linux-gnu-ar
+x86_64-unknown-linux-gnu-dlltool
+x86_64-unknown-linux-gnu-lib
+x86_64-unknown-linux-gnu-link
+x86_64-unknown-linux-gnu-mt
+x86_64-unknown-linux-gnu-objdump
+x86_64-unknown-linux-gnu-ranlib
+x86_64-unknown-linux-gnu-strip
+xdg-open
+xlf
+xlf90
+xlf95
+xmkmf
+X.Org SGML entities >= 1.8
+xpdf
+ansidecl.h usability
+ansidecl.h presence
+ansidecl.h
+nan.h usability
+nan.h presence
+nan.h
+fp_class.h usability
+fp_class.h presence
+fp_class.h
+isnand
+fp_class
+class
+fpclass
+_stat
+ansidecl.h usability
+ansidecl.h presence
+ansidecl.h
+nan.h usability
+nan.h presence
+nan.h
+fp_class.h usability
+fp_class.h presence
+fp_class.h
+isnand
+fp_class
+class
+fpclass
+_stat
+whether g++ supports C++11 features by default
+tsol/label.h usability
+tsol/label.h presence
+tsol/label.h
+is_system_labeled
+launchd
+getlocalename_l
+net/if_dl.h usability
+net/if_dl.h presence
+net/if_dl.h
+sys/disklabel.h usability
+sys/disklabel.h presence
+sys/disklabel.h
+sys/disk.h usability
+sys/disk.h presence
+sys/disk.h
+whether llseek is declared
+whether struct stat has a st_flags field
+chflags
+fadvise64
+getmntinfo
+optreset
+sem_init
+whether we can link with -static
+if building for some Win32 platform
+more warnings
+x86_64-generic-linux-gnu-makeindex
+x86_64-generic-linux-gnu-pdflatex
+x86_64-generic-linux-gnu-egrep
+x86_64-generic-linux-gnu-dvips
+x86_64-generic-linux-gnu-latex
+x86_64-generic-linux-gnu-dot
+x86_64-generic-linux-gnu-perl
+x86_64-generic-linux-gnu-doxygen
+latex
+dvips
+whether more special flags are required for pthreads
+Configure miss: whether to build with gcov testing
+if compiler needs -Werror to reject unknown flags
+pow
+struct stat.st_mtimensec
+struct statvfs.f_basetype
+struct statvfs.f_fstypename
+sys/audioio.h presence
+valgrind/valgrind.h presence
+valgrind/valgrind.h usability
+whether C compiler handles -Werror -Wunknown-warning-option
+whether self tests are run under valgrind
+arc4random_buf
+sys/ucred.h
+sys/utime.h
+valgrind/valgrind.h
+struct statfs.f_fstypename
+whether __argv is declared
+whether <limits.h> defines MIN and MAX
+whether C compiler accepts -Werror=unknown-warning-option
+whether gcc is Clang
+whether more special flags are required for pthreads
+whether the C locale is free of encoding errors
+winsock2.h
+getpeereid
+getpeerucred
+whether wint_t is too small
+xlocale.h usability
+whether the linker accepts -Wl,-fatal_warnings
+whether to build with code coverage support
+xlocale.h presence
+getexecname
+getprogname
+whether stdint.h predates C++11
+xlocale.h
+i686-generic-linux-gnu-pkg-config
+i686-generic-linux-gnu-objdump
+i686-generic-linux-gnu-dlltool
+i686-generic-linux-gnu-mt
+i686-generic-linux-gnu-strip

autospec/docs.py

@@ -1,48 +0,0 @@
-#!/bin/true
-#
-# docs.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import os
-import shutil
-import sys
-
-transforms = {
-    'changes': 'ChangeLog',
-    'changelog.txt': 'ChangeLog',
-    'changelog': 'ChangeLog',
-    'changes.rst': 'ChangeLog',
-    'changes.txt': 'ChangeLog',
-    'news': 'NEWS'
-}
-interests = transforms.keys()
-
-
-def scan_for_changes(download_path, dir):
-    found = []
-    for dirpath, dirnames, files in os.walk(dir, topdown=False):
-
-        hits = [x for x in files if x.lower() in interests and x.lower() not in found]
-        for item in hits:
-            source = os.path.join(dirpath, item)
-            target = os.path.join(download_path, transforms[item.lower()])
-            try:
-                shutil.copy(source, target)
-            except Exception as e:
-                print("Error copying file: %s", e)
-                sys.exit(1)
-            found.append(item)

autospec/download.py

@@ -0,0 +1,79 @@
+#!/usr/bin/true
+#
+# download.py - part of autospec
+# Copyright (C) 2018 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+from io import BytesIO
+
+import pycurl
+from util import print_fatal
+
+
+def do_curl(url, dest=None, post=None, is_fatal=False):
+    """
+    Perform a curl operation for `url`.
+
+    If `post` is set, a POST is performed for `url` with fields taken from the
+    specified value. Otherwise a GET is performed for `url`. If `dest` is set,
+    the curl response (if successful) is written to the specified path and the
+    path is returned. Otherwise a successful response is returned as a BytesIO
+    object. If `is_fatal` is `True` (`False` is the default), a GET failure,
+    POST failure, or a failure to write to the path specified for `dest`
+    results in the program exiting with an error. Otherwise, `None` is returned
+    for any of those error conditions.
+    """
+    c = pycurl.Curl()
+    c.setopt(c.URL, url)
+    if post:
+        c.setopt(c.POSTFIELDS, post)
+    c.setopt(c.FOLLOWLOCATION, True)
+    c.setopt(c.FAILONERROR, True)
+    c.setopt(c.CONNECTTIMEOUT, 10)
+    c.setopt(c.TIMEOUT, 600)
+    c.setopt(c.LOW_SPEED_LIMIT, 1)
+    c.setopt(c.LOW_SPEED_TIME, 10)
+    buf = BytesIO()
+    c.setopt(c.WRITEDATA, buf)
+    try:
+        c.perform()
+    except pycurl.error as e:
+        if is_fatal:
+            print_fatal("Unable to fetch {}: {}".format(url, e))
+            sys.exit(1)
+        return None
+    finally:
+        c.close()
+
+    # write to dest if specified
+    if dest:
+        try:
+            with open(dest, 'wb') as fp:
+                fp.write(buf.getvalue())
+        except IOError as e:
+            if os.path.exists(dest):
+                os.unlink(dest)
+            if is_fatal:
+                print_fatal("Unable to write to {}: {}".format(dest, e))
+                sys.exit(1)
+            return None
+
+    if dest:
+        return dest
+    else:
+        return buf

autospec/files.py

@@ -19,382 +19,410 @@
 # %files section management
 #
 
-import build
-from collections import OrderedDict
-import lang
+import os
 import re
-import tarball
-import buildreq
-import config
-# todo package splits
+from collections import OrderedDict
 
-# per sub-package file list for spec purposes
-packages = OrderedDict()
-main_requires = OrderedDict()
-
-# global file list to weed out dupes
-files = []
-files_blacklist = []
-excludes = []
-extras = []
-setuid = []
-attrs = {}
-
-newfiles_printed = 0
-
-#
-# Do we need ALL include files in a dev package, even if they're not in /usr/include?
-# Yes in the general case, but for example for R packages,
-# the answer is No.
-want_dev_split = 1
+import util
 
 
-def push_main_requires(package):
-    global main_requires
-    if (package not in main_requires):
-        main_requires[package] = set()
+class FileManager(object):
+    """Class to handle spec file %files section management."""
 
+    def __init__(self, config, package):
+        """Set defaults for FileManager."""
+        self.config = config
+        self.package = package
+        self.packages = OrderedDict()  # per sub-package file list for spec purposes
+        self.files = set()  # global file set to weed out dupes
+        self.files_blacklist = set()
+        self.excludes = []
+        self.manual_excludes = []
+        self.file_maps = {}  # Filename-to-package mapping
+        self.setuid = []
+        self.attrs = {}
+        self.locales = []
+        self.newfiles_printed = False
+        # Do we need ALL include files in a dev package, even if they're not in
+        # /usr/include?  Yes in the general case, but for example for R
+        # packages, the answer is No.
+        self.want_dev_split = True
+        self.has_banned = False
 
-def push_package_file(filename, package="main"):
-    global packages
-    global newfiles_printed
+    @staticmethod
+    def banned_path(path):
+        """Check if the path is either banned or in a banned subdirectory."""
+        banned_paths = [r"/etc.*",
+                        r"/opt.*",
+                        r"/usr/local.*",
+                        r"/usr/etc.*",
+                        r"/usr/src.*",
+                        r"/var.*"]
+        for bpath in banned_paths:
+            if re.search(r"^(/V3|/V4|/VA)?" + bpath, path):
+                return True
+        return False
 
-    if (package not in packages):
-        packages[package] = set()
-    packages[package].add(filename)
-    build.must_restart = build.must_restart + 1
-    if newfiles_printed == 0:
-        print("  New %files content found")
-        newfiles_printed = 1
+    def push_package_file(self, filename, package="main"):
+        """Add found %file and indicate to build module that we must restart the build."""
+        if package not in self.packages:
+            self.packages[package] = set()
 
+        if FileManager.banned_path(filename):
+            util.print_warning(f"  Content {filename} found in banned path, skipping")
+            self.has_banned = True
+            return
 
-def file_pat_match(filename, pattern, package, replacement="", prefix=""):
-    if replacement == "":
-        replacement = prefix + filename
+        # prepend the %attr macro if file defined in 'attrs' control file
+        if filename in self.attrs:
+            mod = self.attrs[filename][0]
+            u = self.attrs[filename][1]
+            g = self.attrs[filename][2]
+            filename = "%attr({0},{1},{2}) {3}".format(mod, u, g, filename)
+        self.packages[package].add(filename)
+        self.package.file_restart += 1
+        if not self.newfiles_printed:
+            print("  New %files content found")
+            self.newfiles_printed = True
 
-    pat = re.compile(pattern)
-    match = pat.search(filename)
-    if match:
-        if filename in excludes:
-            push_package_file("%exclude " + filename, package)
+    def compat_exclude(self, filename):
+        """Exclude non-library files if the package is for compatability."""
+        if not self.config.config_opts.get("compat"):
+            return False
+
+        patterns = [
+            r"/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so",
+            r"/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so",
+            r"/usr/lib64/haswell/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/share/package-licenses/"]
+
+        exclude = True
+        for pat in patterns:
+            pat = re.compile(r"^(/V3|/V4|/VA)?" + pat)
+            if pat.search(filename):
+                exclude = False
+                break
+
+        return exclude
+
+    def file_pat_match(self, filename, pattern, package, replacement=""):
+        """Search for pattern in filename.
+
+        Attempt to find pattern in filename, if pattern matches push package file.
+        If that file is also in the excludes list, don't push the file.
+        Returns True if a file was pushed, False otherwise.
+        """
+        if not replacement or self.config.config_opts.get("no_glob"):
+            replacement = filename
+
+        # compat files should always be excluded
+        if self.compat_exclude(filename):
+            self.excludes.append(filename)
             return True
-        push_package_file(replacement, package)
-        return True
-    else:
+
+        # All patterns at this time and should always be prefixed by '^'
+        # but just in case add the following to strip just the '^'
+        pattern = pattern if not pattern.startswith('^') else pattern[1:]
+        pat = re.compile(r"^(/V3|/V4|/VA)?" + pattern)
+        match = pat.search(filename)
+        if match:
+            if len(match.groups()) > 0 and match.groups()[0] in ['/V3', '/V4', '/VA']:
+                norm_filename = filename.removeprefix(match.groups()[0])
+                if replacement != filename:
+                    replacement = match.groups()[0] + replacement
+            else:
+                norm_filename = filename
+            if norm_filename in self.excludes:
+                return True
+
+            self.push_package_file(replacement, package)
+            return True
+
         return False
 
+    def file_is_locale(self, filename):
+        """If a file is a locale, appends to self.locales and returns True, returns False otherwise."""
+        pat = re.compile(r"^/usr/share/locale/.*/(.*)\.mo")
+        match = pat.search(filename)
+        if match:
+            lang = match.group(1)
+            if lang not in self.locales:
+                self.locales.append(lang)
+                print("  New locale:", lang)
+                self.package.must_restart += 1
+                if "locales" not in self.packages:
+                    self.packages["locales"] = set()
 
-def file_is_locale(filename):
-    pat = re.compile("^/usr/share/locale/.*/(.*)\.mo")
-    match = pat.search(filename)
-    if match:
-        l = match.group(1)
-        lang.add_lang(l)
-        return True
-    else:
-        return False
-
-
-def push_file(filename):
-    global files
-    global files_blacklist
-    global extras
-    global setuid
-    global attrs
-    for file in files:
-        if file == filename:
-            return 0
-
-    if filename in files_blacklist:
-        return 0
-    files.append(filename)
-
-    if file_is_locale(filename):
-        return
-
-    # extras
-    if filename in extras:
-        push_package_file(filename, "extras")
-        excludes.append(filename)
-
-    if filename in setuid:
-        fn = "%attr(4755, root, root) " + filename
-        push_package_file(fn, "setuid")
-        excludes.append(filename)
-
-    if filename in attrs:
-        fn = "%s(%s,%s,%s) %s" % (attrs[filename][0], attrs[filename][1], attrs[filename][2],
-                                  attrs[filename][3], filename)
-        push_package_file(fn, "attr")
-        excludes.append(filename)
-
-    if file_pat_match(filename, r"^/usr/share/omf", "main", "/usr/share/omf/*"):
-        return
-
-    if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib", "/usr/lib/*.so.*"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib", "/usr/lib64/*.so.*"):
-        return
-    if file_pat_match(filename, r"^/usr/libexec/", "bin"):
-        return
-    if file_pat_match(filename, r"^/usr/bin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/usr/sbin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/sbin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/bin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/bin/", "bin"):
-        return
-
-    if file_pat_match(filename, r"^/usr/lib/python.*/", "python", "/usr/lib/python*/*"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/python.*/", "python", "/usr/lib64/python*/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/gir-1\.0/.*\.gir\$", "data", "/usr/share/gir-1.0/*.gir"):
-        return
-    if file_pat_match(filename, r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/girepository-1\.0/.*\.typelib\$", "data", "/usr/share/girepository-1.0/*.typelib"):
-        return
-
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hxx", "dev", "/usr/include/*.hxx"):
-        return
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hpp", "dev", "/usr/include/*.hpp"):
-        return
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h\+\+", "dev", "/usr/include/*.h\+\+"):
-        return
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h", "dev", "/usr/include/*.h"):
-        return
-    if file_pat_match(filename, r"^/usr/include/", "dev"):
-        return
-    if want_dev_split > 0 and file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev", "/usr/lib/*.so"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev", "/usr/lib64/*.so"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib/*.a"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib64/*.a"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev", "/usr/lib/pkgconfig/*.pc"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev", "/usr/lib64/pkgconfig/*.pc"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal/*.ac"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal/*.m4"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"):
-        return
-
-    if file_pat_match(filename, r"^/usr/share/doc/" + tarball.name + "/", "doc", "%doc /usr/share/doc/" + tarball.name + "/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/gtk-doc/html", "doc"):
-        return
-    if file_pat_match(filename, r"^/usr/share/info/", "doc", "%doc /usr/share/info/*"):
-        return
-
-    if file_pat_match(filename, r"^/usr/share/man/man0", "doc", "%doc /usr/share/man/man0/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man1", "doc", "%doc /usr/share/man/man1/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man2", "doc", "%doc /usr/share/man/man2/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man3", "doc", "%doc /usr/share/man/man3/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man4", "doc", "%doc /usr/share/man/man4/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man5", "doc", "%doc /usr/share/man/man5/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man6", "doc", "%doc /usr/share/man/man6/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man7", "doc", "%doc /usr/share/man/man7/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man8", "doc", "%doc /usr/share/man/man8/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man9", "doc", "%doc /usr/share/man/man9/*"):
-        return
-
-    if file_pat_match(filename, r"^/etc/systemd/system/.*\.wants/", "active-units"):
-        return
-
-    # now a set of catch-all rules
-    if file_pat_match(filename, r"^/etc/", "config", "", "%config "):
-        return
-    if file_pat_match(filename, r"^/usr/etc/", "config", "", "%config "):
-        return
-    if file_pat_match(filename, r"^/lib/systemd", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/systemd", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/udev/rules.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/modules-load.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/tmpfiles.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/sysusers.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/sysctl.d", "config"):
-        return
-
-    if file_pat_match(filename, r"^/usr/share/", "data"):
-        return
-
-    # finally move any dynamically loadable plugins (not
-    # perl/python/ruby/etc.. extensions) into lib package
-    if file_pat_match(filename, r"^/usr/lib/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"):
-        return
-
-# locale data gets picked up via find_lang
-    if file_pat_match(filename, r"^/usr/share/locale/", "ignore"):
-        return
-
-    if filename in excludes:
-        push_package_file("%exclude " + filename)
-        return
-
-    push_package_file(filename)
-
-
-def remove_file(file):
-    global files
-    global packages
-    global files_blacklist
-    hit = False
-
-    if file in files:
-        files.remove(file)
-        print("File no longer present: %s" % file)
-        hit = True
-    for pkg in packages:
-        if file in packages[pkg]:
-            packages[pkg].remove(file)
-            print("File no longer present in %s: %s" % (pkg, file))
-            hit = True
-    if hit:
-        if file not in files_blacklist:
-            files_blacklist.append(file)
-        build.must_restart = build.must_restart + 1
-
-
-def write_files_header(file):
-    global files
-    global packages
-
-    groups = dict()
-    groups["dev"] = "Development"
-    groups["bin"] = "Binaries"
-    groups["lib"] = "Libraries"
-    groups["doc"] = "Documentation"
-    groups["data"] = "Data"
-
-    deps = dict()
-
-    deps["dev"] = ["lib", "bin", "data"]
-    deps["bin"] = ["data", "config", "setuid", "attr"]
-    deps["lib"] = ["data", "config"]
-
-    provides = dict()
-    provides["dev"] = ["devel"]
-
-    for pkg in sorted(packages):
-        if pkg == "ignore":
-            continue
-        if pkg == "main":
-            continue
-
-        file.write("\n%package " + pkg + "\n")
-        file.write("Summary: " + pkg +
-                   " components for the " + tarball.name + " package.\n")
-        if pkg in groups:
-            file.write("Group: " + groups[pkg] + "\n")
+            return True
         else:
-            file.write("Group: Default\n")
-        for dep in deps.get(pkg, []):
-            if dep in packages:
-                file.write("Requires: " + tarball.name + "-" + dep + "\n")
-        for prov in provides.get(pkg, []):
-            file.write("Provides: " + tarball.name + "-" + prov + "\n")
-        if pkg == "python":
-            if tarball.name != tarball.name.lower():
-                file.write("Provides: %s-python\n" % tarball.name.lower())
-            for req in sorted(buildreq.pythonreqs):
-                if req in buildreq.buildreqs:
-                    file.write("Requires: " + req + "\n")
-        file.write("\n")
-        file.write("%description " + pkg + "\n")
-        file.write(pkg + " components for the " + tarball.name + " package.\n")
-        file.write("\n")
+            return False
 
+    def _clean_dirs(self, root, files):
+        """Do the work to remove the directories from the files list."""
+        res = set()
+        removed = False
 
-def write_files(file):
-    global files
-    global packages
+        directive_re = re.compile(r"(%\w+(\([^\)]*\))?\s+)(.*)")
+        for f in files:
+            # skip the files with directives at the beginning, including %doc
+            # and %dir directives.
+            # autospec does not currently support adding empty directories to
+            # the file list by prefixing "%dir". Regardless, skip these entries
+            # because if they exist at this point it is intentional (i.e.
+            # support was added).
+            if directive_re.match(f):
+                res.add(f)
+                continue
 
-    file.write("\n%files\n")
-    file.write("%defattr(-,root,root,-)\n")
-    if "main" in packages:
-        for filename in sorted(packages["main"]):
-            file.write(filename + "\n")
+            path = os.path.join(root, f.lstrip("/"))
+            if os.path.isdir(path) and not os.path.islink(path):
+                util.print_warning("Removing directory {} from file list".format(f))
+                self.files_blacklist.add(f)
+                removed = True
+            else:
+                res.add(f)
 
-    for pkg in sorted(packages):
-        if pkg == "ignore":
-            continue
-        if pkg == "main":
-            continue
-        if pkg == "locales":
-            continue
-        file.write("\n%files " + pkg + "\n")
-        file.write("%defattr(-,root,root,-)\n")
-        for filename in sorted(packages[pkg]):
-            file.write(filename + "\n")
+        return (res, removed)
 
+    def clean_directories(self, root):
+        """Remove directories from file list."""
+        removed = False
+        for pkg in self.packages:
+            self.packages[pkg], _rem = self._clean_dirs(root, self.packages[pkg])
+            if _rem:
+                removed = True
 
-def write_scriplets(file):
-    global files
-    global packages
+        return removed
 
-    for pkg in sorted(packages):
-        if pkg == "ignore":
-            continue
-        if pkg == "main":
-            continue
-        if pkg == "locales":
-            continue
-        for script in ["post", "pre"]:
-            content = config.read_conf_file("%s.%s" % (pkg, script))
-            if content:
-                file.write("\n%{0} {1}\n".format(script, pkg))
-                file.writelines(content)
+    def push_file(self, filename, pkg_name):
+        """Perform a number of checks against the filename and push the filename if appropriate."""
+        if filename in self.files or filename in self.files_blacklist:
+            return
 
+        self.files.add(filename)
+        if self.file_is_locale(filename):
+            return
 
-def write_main_subpackage_requires(file):
-    global files
-    global packages
-    global main_requires
+        # Explicit file packaging
+        for k, v in self.file_maps.items():
+            for match_name in v['files']:
+                match = re.search(r"^/(V3|V4|VA)", filename)
+                norm_filename = filename if not match else filename.removeprefix(match.group())
+                if isinstance(match_name, str):
+                    if norm_filename == match_name:
+                        self.push_package_file(filename, k)
+                        return
+                elif len('/'.join(match_name)) <= (len(norm_filename) + 1):
+                    # the match name may be 1 longer due to a glob
+                    # being able to match an empty string
+                    if util.globlike_match(norm_filename, match_name):
+                        path_prefix = '/' if not match else match.group()
+                        self.push_package_file(os.path.join(path_prefix, *match_name), k)
+                        return
 
-    for pkg in packages:
-        if pkg == "ignore":
-            continue
-        if pkg == "main":
-            continue
-        if pkg == "dev":
-            continue
-        if pkg == "active-units":
-            continue
-        if pkg == "extras":
-            continue
-        file.write("Requires: " + tarball.name + "-" + pkg + "\n")
+        if filename in self.setuid:
+            if filename in self.attrs:
+                mod = self.attrs[filename][0]
+                u = self.attrs[filename][1]
+                g = self.attrs[filename][2]
+                newfn = "%attr({0},{1},{2}) {3}".format(mod, u, g, filename)
+            else:
+                newfn = "%attr(4755, root, root) " + filename
+            self.push_package_file(newfn, "setuid")
+            return
 
-    for pkg in main_requires:
-        file.write("Requires: " + pkg + "\n")
+        # autostart
+        part = re.compile(r"^/usr/lib/systemd/system/.+\.target\.wants/.+")
+        if part.search(filename) and 'update-triggers.target.wants' not in filename:
+            if filename not in self.excludes:
+                self.push_package_file(filename, "autostart")
+                self.push_package_file("%exclude " + filename, "services")
+                return
+
+        if self.want_dev_split and self.file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"):
+            return
+
+        # Exclude Windows executables and DLLs unless otherwise configured
+        # Can't just skip them because they could be swept up in a python lib wildcard, for example
+        if re.search(r"[^/]+\.(exe|dll)$", filename):
+            if self.config.config_opts.get('allow_exe'):
+                util.print_warning("Allowing {} because allow_exe is true".format(filename))
+            else:
+                util.print_warning("Blocking {} because allow_exe is false".format(filename))
+                self.excludes.append(filename)
+                return
+
+        # if configured to do so, add .so files to the lib package instead of
+        # the dev package. THis is useful for packages with a plugin
+        # architecture like elfutils and mesa.
+        so_dest = 'lib' if self.config.config_opts.get('so_to_lib') else 'dev'
+        so_dest_ompi = 'openmpi' if self.config.config_opts.get('so_to_lib') else 'dev'
+
+        patterns = [
+            # Patterns for matching files, format is a tuple as follows:
+            # (<raw pattern>, <package>, <optional replacement>, <optional prefix>)
+            # order matters, first match wins!
+            (r"^/usr/share/package-licenses/.{1,}/.{1,}", "license"),
+            (r"^/usr/share/man/man2", "dev"),
+            (r"^/usr/share/man/man3", "dev"),
+            (r"^/usr/share/man/", "man"),
+            (r"^/usr/share/pkgconfig/32.*\.pc$", "dev32"),
+            (r"^/usr/share/pkgconfig/", "dev"),
+            (r"^/usr/share/info/", "info"),
+            (r"^/usr/share/abi/", "abi"),
+            (r"^/usr/share/qt5/examples/", "examples"),
+            (r"^/usr/share/qt6/examples/", "examples"),
+            (r"^/usr/share/omf", "main", "/usr/share/omf/*"),
+            (r"^/usr/share/installed-tests/", "tests"),
+            (r"^/usr/libexec/installed-tests/", "tests"),
+            (r"^/usr/lib/rustlib/x86_64-unknown-linux-gnu/lib/[a-zA-Z0-9._+-]+\.rlib", "lib", "/usr/lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib"),
+            (r"^/usr/lib/rustlib/x86_64-unknown-linux-gnu/analysis/[a-zA-Z0-9._+-]+\.json", "lib", "/usr/lib/rustlib/x86_64-unknown-linux-gnu/analysis/*.json"),
+            (r"^/usr/share/clear/optimized-elf/bin", "bin", "/usr/share/clear/optimized-elf/bin*"),
+            (r"^/usr/share/clear/optimized-elf/exec", "libexec", "/usr/share/clear/optimized-elf/exec*"),
+            (r"^/usr/share/clear/optimized-elf/lib", "lib", "/usr/share/clear/optimized-elf/lib*"),
+            (r"^/usr/share/clear/optimized-elf/other", "lib", "/usr/share/clear/optimized-elf/other*"),
+            (r"^/usr/share/clear/optimized-elf/test", "tests", "/usr/share/clear/optimized-elf/test*"),
+            (r"^/usr/share/clear/optimized-elf/", "lib"),
+            (r"^/usr/share/clear/filemap/", "filemap"),
+            (r"^/usr/lib64/openmpi/bin/", "openmpi"),
+            (r"^/usr/lib64/openmpi/share", "openmpi"),
+            (r"^/usr/lib64/openmpi/include/", "dev"),
+            (r"^/usr/lib64/openmpi/lib/[a-zA-Z0-9._+-]*\.so$", so_dest_ompi),
+            (r"^/usr/lib64/openmpi/lib/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/openmpi/lib/[a-zA-Z0-9._+-]*\.so\.", "openmpi"),
+            (r"^/usr/lib64/openmpi/lib/python3.*/", "openmpi"),
+            (r"^/usr/lib64/openmpi/lib/", "dev"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.so\.", "plugins"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.so\.", "lib"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.so\.", "lib32"),
+            (r"^/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so", "lib"),
+            (r"^/usr/lib64/libkdeinit5", "lib"),
+            (r"^/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so", "lib32"),
+            (r"^/usr/lib64/haswell/[a-zA-Z0-9._+-]*\.so\.", "lib"),
+            (r"^/usr/lib64/gobject-introspection/", "lib"),
+            (r"^/usr/libexec/", "libexec"),
+            (r"^/usr/bin/", "bin"),
+            (r"^/usr/sbin/", "bin"),
+            (r"^/sbin/", "bin"),
+            (r"^/bin/", "bin"),
+            (r"^/usr/lib/python3.*/", "python3", "/usr/lib/python3*/*"),
+            (r"^/usr/share/gir-[0-9\.]+/[a-zA-Z0-9._+-]*\.gir", "data", "/usr/share/gir-1.0/*.gir"),
+            (r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"),
+            (r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"),
+            (r"^/usr/share/cmake-3.7/", "data", "/usr/share/cmake-3.7/*"),
+            (r"^/usr/share/cmake-3.8/", "data", "/usr/share/cmake-3.8/*"),
+            (r"^/usr/share/cmake-3.6/", "data", "/usr/share/cmake-3.6/*"),
+            (r"^/usr/share/girepository-1\.0/.*\.typelib\$", "data", "/usr/share/girepository-1.0/*.typelib"),
+            (r"^/usr/include/", "dev"),
+            (r"^/usr/lib64/girepository-1.0/", "data"),
+            (r"^/usr/share/cmake/", "dev"),
+            (r"^/usr/lib/cmake/", "dev"),
+            (r"^/usr/lib64/cmake/", "dev"),
+            (r"^/usr/lib32/cmake/", "dev32"),
+            (r"^/usr/lib/qt5/mkspecs/", "dev"),
+            (r"^/usr/lib/qt6/mkspecs/", "dev"),
+            (r"^/usr/lib64/qt5/mkspecs/", "dev"),
+            (r"^/usr/lib32/qt5/mkspecs/", "dev32"),
+            (r"^/usr/lib64/qt6/mkspecs/", "dev"),
+            (r"^/usr/lib32/qt6/mkspecs/", "dev32"),
+            (r"^/usr/lib/qt5/", "lib"),
+            (r"^/usr/lib/qt6/", "lib"),
+            (r"^/usr/lib64/qt5/", "lib"),
+            (r"^/usr/lib32/qt5/", "lib32"),
+            (r"^/usr/lib64/qt6/", "lib"),
+            (r"^/usr/lib32/qt6/", "lib32"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/libkdeinit5_[a-zA-Z0-9._+-]*\.so$", "lib"),
+            (r"^/usr/lib32/libkdeinit5_[a-zA-Z0-9._+-]*\.so$", "lib32"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.so$", so_dest + '32'),
+            (r"^/usr/lib64/glibc-hwcaps/x86-64-v[0-9]+/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/haswell/avx512_1/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/haswell/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/haswell/avx512_1/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.a$", "staticdev32"),
+            (r"^/usr/lib/haswell/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/glibc-hwcaps/x86-64-v[0-9]+/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/haswell/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/haswell/avx512_1/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib32/haswell/[a-zA-Z0-9._+-]*\.a$", "staticdev32"),
+            (r"^/usr/lib/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib64/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib32/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev32"),
+            (r"^/usr/lib64/glibc-hwcaps/x86-64-v[0-9]+/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib64/haswell/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib64/haswell/avx512_1/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.la$", "dev"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.la$", "dev"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.la$", "dev32"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.prl$", "dev"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.prl$", "dev"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.prl$", "dev32"),
+            (r"^/usr/share/aclocal/[a-zA-Z0-9._+-]*\.ac$", "dev", "/usr/share/aclocal/*.ac"),
+            (r"^/usr/share/aclocal/[a-zA-Z0-9._+-]*\.m4$", "dev", "/usr/share/aclocal/*.m4"),
+            (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9._+-]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"),
+            (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9._+-]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"),
+            (r"^/usr/share/doc/" + re.escape(pkg_name) + "/", "doc", "/usr/share/doc/" + re.escape(pkg_name) + "/*"),
+            (r"^/usr/share/doc/", "doc"),
+            (r"^/usr/share/gtk-doc/html", "doc"),
+            (r"^/usr/share/help", "doc"),
+            (r"^/usr/share/info/", "doc", "/usr/share/info/*"),
+            # now a set of catch-all rules
+            (r"^/lib/systemd/system/", "services"),
+            (r"^/lib/systemd/user/", "services"),
+            (r"^/usr/lib/systemd/system/", "services"),
+            (r"^/usr/lib/systemd/user/", "services"),
+            (r"^/usr/lib/udev/hwdb.d", "config"),
+            (r"^/usr/lib/udev/rules.d", "config"),
+            (r"^/usr/lib/modules-load.d", "config"),
+            (r"^/usr/lib/tmpfiles.d", "config"),
+            (r"^/usr/lib/sysusers.d", "config"),
+            (r"^/usr/lib/sysctl.d", "config"),
+            (r"^/usr/share/", "data"),
+            (r"^/usr/lib/perl5/", "perl", "/usr/lib/perl5/*"),
+            # finally move any dynamically loadable plugins (not
+            # perl/python/etc.. extensions) into lib package
+            (r"^/usr/lib/.*/[a-zA-Z0-9._+-]*\.so", "lib"),
+            (r"^/usr/lib64/.*/[a-zA-Z0-9._+-]*\.so", "lib"),
+            (r"^/usr/lib32/.*/[a-zA-Z0-9._+-]*\.so", "lib32"),
+            # locale data gets picked up via file_is_locale
+            (r"^/usr/share/locale/", "ignore")]
+
+        for pat_args in patterns:
+            if self.file_pat_match(filename, *pat_args):
+                return
+
+        if filename in self.excludes:
+            return
+
+        self.push_package_file(filename)
+
+    def remove_file(self, filename):
+        """Remove filename from local file list."""
+        hit = False
+
+        if filename in self.files:
+            self.files.remove(filename)
+            print("File no longer present: {}".format(filename))
+            hit = True
+        for pkg in self.packages:
+            if filename in self.packages[pkg]:
+                self.packages[pkg].remove(filename)
+                print("File no longer present in {}: {}".format(pkg, filename))
+                hit = True
+        if hit:
+            self.files_blacklist.add(filename)
+            self.package.must_restart += 1
+
+    def load_specfile(self, specfile):
+        """Load a specfile instance with relevant information to be written to the spec file."""
+        specfile.packages = self.packages
+        specfile.excludes = self.excludes
+        specfile.locales = self.locales
+        specfile.file_maps = self.file_maps
+        specfile.setuid = self.setuid

autospec/git.py

@@ -19,51 +19,149 @@
 # Commit to git
 #
 
+import glob
+import os
 import subprocess
+import sys
+import tempfile
 
-import build
-import buildpattern
-import tarball
-import test
-import docs
-from util import call
-import config
+from util import call, open_auto, write_out
 
 
-def commit_to_git(path):
-
-    if not config.git_uri:
-        return
-
-    call("git init", stdout=subprocess.DEVNULL, cwd=path)
-
+def get_autospec_info():
+    """Get the latest tag from autospec."""
+    path = os.path.dirname(sys.path[0])
+    git_out = tempfile.mkstemp()[1]
     try:
-        call("git config --get remote.origin.url", cwd=path)
-    except subprocess.CalledProcessError:
-        upstream_uri = config.git_uri % {'NAME': tarball.name}
-        call("git remote add origin %s" % upstream_uri, cwd=path)
+        call("git tag -l --sort=-v:refname", git_out, cwd=path)
+        with open_auto(git_out) as gfile:
+            tag = gfile.readlines()[0].strip()
+    except Exception:
+        tag = ""
+    os.unlink(git_out)
+    try:
+        call('git log -1 --pretty=format:"%h"', git_out, cwd=path)
+        with open_auto(git_out) as gfile:
+            commit = gfile.readlines()[0].strip()
+    except Exception:
+        commit = ""
+    os.unlink(git_out)
+    return tag, commit
+
+
+def commit_to_git(config, name, success):
+    """Update package's git tree for autospec managed changes."""
+    path = config.download_path
+    call("git init -b main", stdout=subprocess.DEVNULL, cwd=path)
+
+    # This config is used for setting the remote URI, so it is optional.
+    if config.git_uri:
+        try:
+            call("git config --get remote.origin.url", cwd=path)
+        except subprocess.CalledProcessError:
+            upstream_uri = config.git_uri % {'NAME': name}
+            call("git remote add origin %s" % upstream_uri, cwd=path)
 
     for config_file in config.config_files:
-        call("git add %s" % config_file, cwd=path)
-    for unit in buildpattern.sources["unit"]:
+        call("git add %s" % config_file, cwd=path, check=False)
+    for unit in config.sources["unit"]:
         call("git add %s" % unit, cwd=path)
     call("git add Makefile", cwd=path)
     call("git add upstream", cwd=path)
-    call("git add *.spec", cwd=path)
-    if test.unit_pass_written:
-        call("git add unit_tests_must_pass", cwd=path)
-    call("git add %s.tmpfiles" % tarball.name, check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add funroll-loops", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add *.spec'", cwd=path)
+    call("git add %s.tmpfiles" % name, check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add %s.sysusers" % name, check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add prep_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add pypi.json", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add build_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add make_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add install_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add series", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    # Add/remove version specific patch lists
+    for filename in glob.glob('series.*'):
+        base, version = filename.split('.', 1)
+        if version in config.versions:
+            call("git add {}".format(filename), check=False, stderr=subprocess.DEVNULL, cwd=path)
+        else:
+            call("git rm {}".format(filename), check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.asc'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.sig'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.sha256'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.sign'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.pkey'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure32", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure64", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure_avx2", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure_avx512", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add make_check_command", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add keepstatic", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add *.patch", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    for item in docs.transforms.values():
+    call("bash -c 'shopt -s failglob; git add *.patch'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add *.nopatch'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    for item in config.transforms.values():
         call("git add {}".format(item), check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add release", cwd=path)
+    call("git add symbols", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add symbols32", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add used_libs", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add used_libs32", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add testresults", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add profile_payload", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add options.conf", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure_misses", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add etc_files", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add whatrequires", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add description", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add attrs", check=False, stderr=subprocess.DEVNULL, cwd=path)
 
-    if build.success == 0:
+    # remove deprecated config files
+    call("git rm make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm prep_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm use_clang", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm use_lto", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm use_avx2", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm fast-math", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm broken_c++", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm skip_test_suite", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm optimize_size", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm asneeded", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm broken_parallel_build", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm pgo", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm unit_tests_must_pass", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm funroll-loops", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm keepstatic", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm allow_test_failures", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm no_autostart", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm insecure_build", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm conservative_flags", check=False, stderr=subprocess.DEVNULL, cwd=path)
+
+    # add a gitignore
+    ignorelist = [
+        ".*~",
+        "*~",
+        "*.info",
+        "*.mod",
+        "*.swp",
+        ".repo-index",
+        "*.log",
+        "build.log.round*",
+        "*.tar.*",
+        "*.tgz",
+        "!*.tar.*.*",
+        "*.zip",
+        "*.jar",
+        "*.pom",
+        "*.xml",
+        "commitmsg",
+        "results/",
+        "rpms/",
+        "for-review.txt",
+        ""
+    ]
+    write_out(os.path.join(path, '.gitignore'), '\n'.join(ignorelist))
+    call("git add .gitignore", check=False, stderr=subprocess.DEVNULL, cwd=path)
+
+    if success == 0:
         return
 
     call("git commit -a -F commitmsg ", cwd=path)

autospec/ignored_commands

@@ -0,0 +1,599 @@
+# This file contains ignored commands from configure output
+# in the form: <configure line>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+# For patterns that start with '#', escape the '#' as '\#'.
+-Wmissing-braces
+-Wparentheses-equality
+-Wtypedef-redefinition
+-Wunknown-attributes
+-Wunknown-warning-option
+-lc should be explicitly linked in
+-ldld
+-lmkl
+-lposix1e
+-lsec
+-lselinux
+-lsmack
+-lsocket
+-lxpg4
+// is distinct from /
+/dev/ptc
+/etc/SuSE-release
+/etc/debian_version
+/etc/fedora-release
+/etc/gentoo-release
+/etc/mandriva-release
+/etc/redhat-release
+<limits.h> defines MIN and MAX
+<pthread.h> pollutes the namespace
+AIX ACL
+ANSI C header files
+ARM NEON in current arch/CFLAGS
+Avahi
+AvailabilityMacros.h
+BeOS
+C compiler accepts -Wcomma
+C compiler accepts -Werror=unknown-warning-option
+C compiler accepts -Winitializer-overrides
+C compiler handles -Werror -Wunknown-warning-option
+C symbols are prefixed with underscore at the linker level
+CC
+CDPARANOIA
+CELT051
+CFLocaleCopyCurrent
+CFPreferencesCopyAppValue
+CYGWIN or MSYS environment
+Darwin (Mac OS X)
+Darwin (Mac OS X) platform
+ELOGIND
+GTK_QUARTZ
+HAVE_BIG_ENDIAN
+HAVE_INCOHERENT_MMAP
+HDF5 library has enforced version 1.6 API
+HP-UX
+KF5
+LIBPSL
+LIBTEAMDCTL
+LUA_PREFIX
+Minix Amsterdam compiler
+MobileCoreServices/MobileCoreServices.h
+OS.h
+QNX
+Qt5WebKitWidgets
+SELINUX
+Solaris 11.4 locale system
+Solaris ACL
+Ubuntu
+Unknown pattern match:
+WIN32
+Win32
+Win32 platform in general
+XATTR_NAME_SMACKEXEC in linux/xattr.h
+XATTR_NOFOLLOW
+XML catalog (/etc/xml/catalog)
+XSI (rather than GNU) prototype for strerror_r
+_ prefix in compiled symbols
+_FILE_OFFSET_BITS value needed for large files
+_FILE_OFFSET_BITS value needed large files
+_LARGEFILE_SOURCE value needed for large files
+_LARGEFILE_SOURCE value needed large files
+_NSGetEnviron
+__ApplePrivate_gss_acquire_cred_ex_f
+__INTEL_COMPILER
+__SUNPRO_C
+__argv
+__clang__
+__freadahead
+__freadptr
+__freadptrinc
+__fseterr
+__i386__
+__mktime_internal
+__popcnt
+__popcnt64
+__res_init
+__secure_getenv
+__strtoll
+__strtoull
+__sys_siglist
+__uint128_t
+__xpg4
+_byteswap_ulong
+_byteswap_ushort
+_doprnt
+_fseeki64
+_getpid
+_putenv_s
+_set_invalid_parameter_handler
+_snprintf
+_strdup
+_xpg4_setrunelocale in -lxpg4
+a
+a statically linked program can dlopen itself
+a turkish Unicode locale
+abs
+acl_get_file in -lposix1e
+acos
+acosh
+any prototype needed for pread
+any prototype needed for pwrite
+ar
+arithmetic hrtime_t
+as
+asin
+asinh
+atan
+atanh
+atomic_add_32 compiler builtin
+bcopy
+binary relocation should be enabled
+broken poll
+bstring.h
+build strict
+byte ordering is bigendian
+c++
+cbrt
+cc
+ceil
+chsize
+clang option to accept ISO C89
+clock_gettime
+closedir returns void
+closefrom
+compiler driver understands Ada
+compilers -Wno-format-truncation
+compilers -Wno-tautological-compare
+compilers -Wno-unused-result
+compilers -Wsign-conversion
+cos
+cosh
+crt_externs.h
+crtdefs.h
+crypt
+crypt_r
+cygwin
+declaration of PTHREAD_MUTEX_ROBUST
+default print group
+default print user
+dependency style of gcc
+desired default level of POSIX conformance
+directio
+dirfd is a macro
+dl.h
+dlclose
+dld_link in -ldld
+dlerror
+dlltool
+dlopen
+dlopen()
+dlsym
+dns_sd.h
+dustat.h
+eatmydata
+epydoc
+erf
+erfc
+errno_t
+exp
+explicit OBJCFLAGS
+external symbol _system_configuration
+extra flags to get ANSI library prototypes
+fabs
+fake locale system (OpenBSD)
+fallocate
+fdatasync
+fdatasync... (cach
+fdwalk
+fflush works on input streams
+filefrag
+floor
+fop
+for gcc option to accept ISO C89
+for minix/config.h
+for mt
+for sysroot
+for x86_64-generic-linux-gnu-dlltool
+for x86_64-generic-linux-gnu-mt
+for x86_64-generic-linux-gnu-objdump
+for x86_64-generic-linux-gnu-strip
+fpurge
+fpurge without a macro
+frame.h
+g++ option to enable C++11 features
+gai_strerrorA
+gcc
+gcc accepts -g
+gcc is Clang
+gcc needs -traditional
+gcc option to accept ANSI C
+gcc option to accept ISO C89
+gcc option to accept ISO C99
+gcc option to enable C11 features
+gcc options needed to detect all undeclared functions
+gcc understands --disable-instructions
+gcc understands --omg-optimized
+gcc understands -Wold-style-definition
+gcc understands -Wstrict-prototypes
+gccgo
+gccs -Wall
+gccs -mpower8-vector
+gdiff
+getaddrinfo
+getattrat
+getcwd
+getcwd aborts when 4k < cwd_length < 16k
+getdents
+getexecname
+getgrent_nomembers
+getgrgid_nomembers
+getgrnam_nomembers
+gethostbyaddr
+gethostbyname
+gethostbyname... (cach
+gethrtime
+getifaddrs
+getifaddrs() and AF_LINK
+getlocalename_l
+getmntent
+getmntinfo
+getnameinfo
+getpeereid
+getpeerucred
+getppriv
+getprogname
+getpseudotty
+gets without a macro
+getservbyname
+getservbyname... (cach
+getspent
+getsysinfo
+gettimeofday clobbers localtime buffer
+gm4
+gnatbind
+gnatmake
+gnudiff
+gnum4
+gtar
+history.h
+how to do getaddrinfo, freeaddrinfo and getnameinfo... checking for getaddrinfo
+how to run the C preprocessor
+hstrerror
+hurd.h
+hypot
+icc
+iconv_open
+ieeefp.h
+if -lm is required for maths functions
+if : is a manifest tool
+if GCC is actually Clang
+if X11 header
+if X11 header files implicitly declare return values
+if building for some Win32 platform
+if building the Windows installer
+if character set is EBCDIC
+if compiler needs -Werror to reject unknown flags
+if compiler supports -R
+if compiling for Mac OS X
+if compiling for Win32
+if compiling with clang
+if debug is enabled
+if g++ static flag -static works
+if gcc static flag -static works
+if gcc supports -fno-rtti -fno-exceptions
+if gccs -Werror=unknown-warning-option
+if gccs -Werror=unused-command-line-argument
+if gccs -fno-rtti -fno-exceptions
+if internal cryptsetup PBKDF2 is compiled-in
+if more special flags are required for pthreads
+if zts is enabled
+included gettext is requested
+inet_ntop
+inet_ntop... (cach
+inet_pton
+install_name_tool
+integer division by zero raises SIGFPE
+io.h
+ioctl with POSIX signature
+is_selinux_enabled in -lselinux
+isapipe
+j0
+j1
+kqueue
+launch.h
+launch_activate_socket
+lchmod
+ld
+libc.h
+libiconv_open
+libnet.h version 1.1.x
+library containing opendir
+library containing strerror
+library setproctitle
+libsigsegv
+libunwind-ia64.h
+libutil.h
+link(2) dereferences a symlink
+lipo
+listmntent
+listmntent of Cray/Unicos-9
+localtime caches TZ
+localtime loops forever near extrema
+log
+log10
+login_tty
+long double and double are the same
+lstat accepts an empty string
+mach-o/dyld.h
+mach/host_info.h
+mach/mach_time.h
+machine/hal_sysinfo.h
+main in -lelf
+major_t
+matchpathcon_init_prefix
+mbslen
+mbswidth in <wchar.h>
+member __ss_family in struct sockaddr_storage
+member sa_len in struct sockaddr
+member sin6_len in struct sockaddr_in6
+member sin_len in struct sockaddr_in
+memcpy
+memory.h
+memset_s
+microuptime
+minix/config.h
+minix/config.h presence
+minix/config.h usability
+minor_t
+mntctl function and struct vmount
+mode_to_security_class
+more special flags are required for pthreads
+more warnings
+mt
+multithread API to use
+mxml.h
+nanosleep
+nanotime
+nanouptime
+native Win32
+native macOS
+net/bpf.h
+net/if_dl.h
+net/pfilt.h
+net/pfvar.h
+net/raw.h
+nfs/vfs.h
+nm
+objc
+objcc
+objcopy
+objdump
+obstacks that work with any size object
+offset_t
+opendir
+openpty
+org
+plugin documentation
+port_create
+posix_spawn
+precise C stack overflow detection
+printfs the 'n' directive
+priv.h
+process.h
+pstat_getdynamic
+pstat_getstatic
+pthread_create
+pthread_create using -lpthreads
+pthread_mutex_consistent
+pthread_mutexattr_setrobust
+pthread_np.h
+pthread_rwlock_rdlock prefers a writer to a reader
+pthread_setname_np(const char*)
+pthread_sigmask
+pthread_sigmask is only a macro
+pthreads work with -Kthread
+pthreads work with -kthread
+pthreads work without any flags
+putmsg in -lstr
+py.test-2.7
+pyinotify
+python2.6
+ranlib
+readelf
+removefile
+res_search
+rint
+rresvport_af
+sched_getaffinity_np
+selinux/context.h
+selinux/flask.h
+selinux/selinux.h
+setfilecon
+setproctitle
+setproctitle_init
+setsid
+setsockopt
+shared libraries
+shl_findsym
+shl_load
+shl_load in -ldld
+shl_unload
+sig2str
+sighandler_t
+sighandlers must be reinstalled
+sigsetjmp
+sigvec
+sinh
+size_t
+smack_new_label_from_self in -lsmack
+snprintf
+socket
+socket in -lsocket
+socketpair
+some Darwin platform
+some Win32 platform
+special C compiler options needed for large files
+special C compiler options needed large files
+sqrt
+st_dm_mode in struct stat
+st_gen member in stat structure
+stack_t has an ss_base field
+stat accepts an empty string
+stat() ignores a trailing slash
+statfs that truncates block counts
+static libraries
+stdint.h
+stdint.h predates C++11
+strerror
+strerror_r with POSIX signature
+strerror_s
+stricmp
+strip
+strlcat
+strlcpy
+strmode
+strncpy_s
+strnicmp
+strnstr
+struct dirent.d_namlen
+struct fsstat.f_fstypename
+struct sockaddr.sa_len
+struct sockaddr_in.sin_len
+struct sockaddr_in6.sin6_len
+struct sockaddr_storage.__ss_family
+struct stat
+struct stat.st_atimensec
+struct stat.st_author
+struct stat.st_birthtim.tv_nsec
+struct stat.st_birthtime
+struct stat.st_birthtimensec
+struct stat.st_birthtimespec.tv_nsec
+struct stat.st_ctimensec
+struct stat.st_flags
+struct stat.st_mtimensec
+struct stat.st_mtimespec.tv_nsec
+struct statfs.f_fstypename
+struct statfs.f_namemax
+struct statvfs.f_basetype
+struct statvfs.f_fstypename
+struct statvfs.f_type
+struct tm.__tm_gmtoff
+struct utmp.ut_exit.ut_exit
+struct utmp.ut_exit.ut_termination
+struct utmpx.ut_exit.ut_exit
+struct utmpx.ut_exit.ut_termination
+suncc
+symbols are prefixed
+sys/access.h
+sys/dirent.h
+sys/dlpi.h
+sys/endian.h
+sys/event.h
+sys/filio.h
+sys/fs/s5param.h
+sys/fs_types.h
+sys/ioccom.h
+sys/loadavg.h
+sys/mkdev.h
+sys/mntent.h
+sys/mnttab.h
+sys/pstat.h
+sys/ptem.h
+sys/sockio.h
+sys/stream.h
+sys/sysmp.h
+sys/systemcfg.h
+sys/systeminfo.h
+sys/table.h
+sys/thr.h
+sys/types.h defines makedev
+sys/ucred.h
+sys/utime.h
+sysctl
+sysctlbyname
+sysmouse
+sysmp
+sysroot
+system header files limit the line length
+table
+tan
+tanh
+termios.h defines TIOCGWINSZ
+termios.h needs _XOPEN_SOURCE
+tgetflag
+the BeOS
+the C locale is free of encoding errors
+the compiler is actually Clang
+the linker accepts -Wl,-fatal_warnings
+the pthreads library -llthread
+the pthreads library -lpthreads
+the systems xlc include
+thr_self
+timezone_t
+to add custom build version
+to enable debugging
+to enable maintainer-specific portions of Makefiles
+to enable pch feature
+to sign windows build
+to treat the installation as read-only
+to turn on debugging
+to turn on profiling
+to turn warnings to errors
+to use C++
+to use statvfs64
+toupper
+tputs() uses outfuntype
+tzset clobbers localtime buffer
+uint_t
+unistd.h
+use of TIOCGWINSZ requires termios.h
+use of struct winsize requires sys/ptem.h
+util.h
+util/debug.h
+util/msg18n.h
+va_lists can be copied by value
+varargs.h
+variable rl_event_hook
+vasnprintf
+vfork.h
+w32api/winbase.h
+w32api/wtypes.h
+w3m
+we are cross compiling
+we are using the GNU C compiler
+we are using the GNU Objective C compiler
+we are using the GNU Objective C++ compiler
+we build a relocatable package
+we need to force -D_FILE_OFFSET_BITS=64
+we want to fetch tarballs
+wget
+whether -Wmissing-braces works as expected
+whether -Wparentheses-equality works as expected
+whether -Wtypedef-redefinition works as expected
+whether -Wunknown-attributes works as expected
+whether -Wunknown-warning-option works as expected
+whether -lc should be explicitly linked in
+whether to build static libraries
+whether we are cross compiling
+which package format to use
+wincrypt.h
+windmc
+windows.h
+windres
+winsock2.h
+wint_t is too small
+with code coverage
+with code coverage support
+with gcov testing
+working alloca.h
+working mmap
+working nanosleep
+ws2tcpip.h
+x86_64-generic-linux-gnu-dlltool
+x86_64-generic-linux-gnu-mt
+x86_64-generic-linux-gnu-objdump
+x86_64-generic-linux-gnu-strip
+xlocale.h
+xmkmf
+y0
+y1

autospec/keyid_blocklist

@@ -0,0 +1 @@
+59FCF207FEA7F445

autospec/lang.py

@@ -1,52 +0,0 @@
-#!/bin/true
-#
-# lang.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Parse config files
-#
-
-import files
-
-
-locales = []
-
-
-def add_lang(lang):
-    global locales
-    if lang in locales:
-        return
-    locales.append(lang)
-    print("  New locale:", lang)
-
-    if "locales" in files.packages:
-        return
-    files.packages["locales"] = []
-
-
-def write_find_lang(file):
-    for lang in locales:
-        file.write("%find_lang " + lang + "\n")
-
-
-def write_lang_files(file):
-    global locales
-    if len(locales) == 0:
-        return
-    file.write("\n%files locales ")
-    for lang in locales:
-        file.write("-f " + lang + ".lang ")
-    file.write("\n%defattr(-,root,root,-)\n\n")

autospec/license.py

@@ -22,256 +22,190 @@
 #
 
 import os
-import tarball
-import urllib
-import config
+import re
+import shlex
+import sys
+import urllib.parse
+
+import chardet
+import download
+import util
 
 default_license = "TO BE DETERMINED"
 
-LICENSE_HASH_FILE = "common/licenses"
-
 licenses = []
-
-license_translations = {
-    "GPL-2.0": "GPL-2.0",
-    "GPLv2": "GPL-2.0",
-    "GPLV2": "GPL-2.0",
-    "GPLV3": "GPL-3.0",
-    "GPL-2": "GPL-2.0",
-    "GPL-2+": "GPL-2.0+",
-    "GPL-2.0+": "GPL-2.0+",
-    "GPLv2+": "GPL-2.0+",
-    "GPL(>=2)": "GPL-2.0+",
-    "GPL(>=-2)": "GPL-2.0+",
-    "LGPL(>=2)": "LGPL-2.0+",
-    "LGPL(>=-2)": "LGPL-2.0+",
-    "LGPLv2": "LGPL-2.0",
-    "LGPLv2.1": "LGPL-2.1",
-    "LGPLv2+": "LGPL-2.1+",
-    "LGPL-2.0+": "LGPL-2.0+",
-    "LGPL-2.1": "LGPL-2.1",
-    "LGPLv2.1": "LGPL-2.1",
-    "LGPL-2.1+": "LGPL-2.1+",
-    "LGPLv2.1+": "LGPL-2.1+",
-    "LGPLv3+": "LGPL-3.0+",
-    "LGPLv3": "LGPL-3.0",
-    "GPL-3.0": "GPL-3.0",
-    "GPLv3": "GPL-3.0",
-    "GPL-3": "GPL-3.0",
-    "LGPL-3": "LGPL-3.0",
-    "zlib": "Zlib",
-    "zlib/libpng": "zlib-acknowledgement",
-    "Boost": "BSL-1.0",
-    "GPL-3.0+": "GPL-3.0+",
-    "GPLv3+": "GPL-3.0+",
-    "GPL3": "GPL-3.0",
-    "GPL(>=3)": "GPL-3.0+",
-    "http://opensource.org/licenses/MIT": "MIT",
-    "mit": "MIT",
-    "http://www.apache.org/licenses/LICENSE-2.0": "Apache-2.0",
-    "Apache License, Version 2.0": "Apache-2.0",
-    "Apache License 2.0": "Apache-2.0",
-    "APL2.0": "Apache-2.0",
-    "APL2": "Apache-2.0",
-    "ASL 2.0": "Apache-2.0",
-    "ASL-2.0": "Apache-2.0",
-    "APL-2.0": "Apache-2.0",
-    "ASL-2": "Apache-2.0",
-    "Apache2.0": "Apache-2.0",
-    "Apache 2.0": "Apache-2.0",
-    "Apache": "Apache-2.0",
-    "apache": "Apache-2.0",
-    "Apache-2": "Apache-2.0",
-    "artistic_2": "Artistic-2.0",
-    "MPLv1.1": "MPL-1.1",
-    "ZPL 2.1": "ZPL-2.1",
-    "ZPL": "ZPL-2.0",
-    "http://creativecommons.org/licenses/BSD/": "BSD-2-Clause",
-    "BSD_3_clause": "BSD-3-Clause",
-    "perl": "Artistic-1.0-Perl",
-    "PSF": "Python-2.0",
-    "Python": "Python-2.0",
-    "BSD_2_clause": "BSD-2-Clause",
-    "Expat": "MIT",
-    "VIM": "Vim"
-}
-
-license_blacklist = {
-    "and": True,
-    "BSD": True,
-    "3BSD": True,
-    "LGPL": True,
-    "GPL": True,
-    "ASL": True,
-    "2.0": True,
-    "advertising": True,
-    "LGPL+BSD": True,
-    "UN": True,
-    "GNU": True,
-    "new": True,
-    "none": True,
-    "License": True,
-    "License": True,
-    "license": True,
-    "Standard": True,
-    "PIL": True,
-    "Software": True,
-    "|": True,
-    "+": True,
-    "UNKNOWN": True,
-    "unknown": True,
-    "BSD-like": True,
-    "or": True,
-    "Modified": True,
-    "3-clause": True,
-    "Unlimited": True,
-    "BSD_3_clause": True,
-    "BSD(3": True,
-    "clause)": True,
-    "GFDL": True,
-    "MPL": True,
-    "Muddy-MIT": True,
-    "LGPL/MIT": True,
-    "License,": True,
-    "LICENCE": True,
-    "-": True,
-    "See": True,
-    "details": True,
-    "for": True,
-    "Version-2.0": True,
-    "exceptions": True,
-    "http://nmap.org/man/man-legal.html": True,
-    "with": True,
-    "style": True,
-    "Foundation": True,
-    "~": True,
-    "open_source": True,
-    "BSDish": True,
-    "EPL": True,
-    "Artistic": True,
-    "(specified": True,
-    "using": True,
-    "classifiers)": True,
-    "APL-2.0": True,
-    "dual": True,
-    "GPL/BSD": True
-}
+license_files = []
+hashes = dict()
 
 
-def add_license(lic):
-    global licenses
-    lic = lic.strip()
+def process_licenses(lics, translations, blacklist):
+    """Handle licenses string from the license server.
 
-    # Bail if the license isn't one we recognize, is a blacklisted word or has already been added
-    if lic in licenses:
-        return False
-    if lic in license_blacklist:
-        return False
-    if lic not in license_translations:
-        licenses.append(lic)
-        return True
-    if license_translations[lic] in licenses:
-        return False
-
-    licenses.append(license_translations[lic])
-    return True
+    The license server response may contain multiple space-separated licenses.
+    Add each license individually.
+    """
+    for lic in lics.split():
+        add_license(lic, translations, blacklist)
 
 
-def license_from_copying_content(copying):
-    with open(copying, 'r', encoding="latin-1") as content_file:
-        content = content_file.read()
-    if content.find("Version 2, June 1991") >= 0:
-        add_license("GPL-2.0")
-    if content.find("Version 3, 29 June 2007") >= 0:
-        add_license("GPL-3.0")
-    if content.find("Version 2.1, February 1999") >= 0:
-        add_license("LGPL-2.1")
-    if content.find("Source Code is licensed under MIT license") >= 0:
-        add_license("MIT")
-    if content.find("Version 2.0, January 2004") >= 0:
-        add_license("Apache-2.0")
+def add_license(lic, translations, blacklist):
+    """Add licenses from the server.
+
+    Add license from license string lic after checking for duplication or
+    presence in the blacklist. Returns False if no license were added, True
+    otherwise.
+    """
+    lic = lic.strip().strip(',')
+    result = False
+
+    # Translate the license if a translation exists
+    real_lic_str = translations.get(lic, lic)
+    real_lics = real_lic_str.split()
+    for real_lic in real_lics:
+        if real_lic in blacklist:
+            continue
+        elif real_lic in licenses:
+            result = True
+        else:
+            result = True
+            licenses.append(real_lic)
+
+    return result
 
 
-def license_from_copying_hash(copying):
-    """Add licenses based on the hash of the copying file"""
-    licenses_list = []
-    hash_sum = tarball.get_sha1sum(copying)
+def decode_license(license):
+    """Try and decode the license string."""
+    def try_with_charset(license, charset):
+        if not charset:
+            return
 
-    licenses_dict = dict()
+        try:
+            return license.decode(charset)
+        except UnicodeDecodeError:
+            if charset in ('ISO-8859-1', 'ISO-8859-15'):
+                if b'\xff' in license:
+                    return try_with_charset(license, 'ISO-8859-13')
+                if b'\xd2' in license and b'\xd3' in license:
+                    return try_with_charset(license, 'mac_roman')
+
+    return try_with_charset(license, chardet.detect(license)['encoding'])
+
+
+def license_from_copying_hash(copying, srcdir, config, name):
+    """Add licenses based on the hash of the copying file."""
+    try:
+        data = util.get_contents(copying)
+    except FileNotFoundError:
+        # LICENSE file is a bad symlink (qemu-4.2.0!)
+        return
+
+    if data.startswith(b'#!'):
+        # Not a license if this is a script
+        return
+
+    data = decode_license(data)
+    if not data:
+        return
+
+    hash_sum = util.get_sha1sum(copying)
 
     if config.license_fetch:
-        with open(copying, "r", encoding="latin-1") as myfile:
-            data = myfile.read()
-            url = config.license_fetch
-            values = {'hash': hash_sum, 'text': data, 'package': tarball.name}
-            data = urllib.parse.urlencode(values)
-            data = data.encode('utf-8')
-            req = urllib.request.Request(url, data)
-            response = urllib.request.urlopen(req)
-            the_page = response.read().decode('utf-8')
-            if len(the_page.strip()) > 0:
-                print("License     : ", the_page.strip(), " (server) (", hash_sum, ")")
-                add_license(the_page.strip())
-                return
-    else:
-        if os.path.exists(LICENSE_HASH_FILE):
-            with open(LICENSE_HASH_FILE, "r") as file:
-                licenses_list = file.readlines()
-        licenses_dict = dict(license.split(" | ") for license in licenses_list)
+        values = {'hash': hash_sum, 'text': data, 'package': name}
+        data = urllib.parse.urlencode(values)
+        data = data.encode('utf-8')
 
-    if hash_sum in licenses_dict:
-        add_license(licenses_dict[hash_sum])
+        buffer = download.do_curl(config.license_fetch, post=data, is_fatal=True)
+        response = buffer.getvalue()
+        page = response.decode('utf-8').strip()
+        if page:
+            print("License     : ", page, " (server) (", hash_sum, ")")
+            process_licenses(page, config.license_translations, config.license_blacklist)
+
+            if page != "none":
+                # Strip the build source directory off the front
+                lic_path = copying[len(srcdir):]
+                # Strip any leading slashes
+                while lic_path.startswith('/'):
+                    lic_path = lic_path[1:]
+                lic_path = shlex.quote(lic_path)
+                license_files.append(lic_path)
+                hashes[lic_path] = hash_sum
+
+            return
+
+    if hash_sum in config.license_hashes:
+        add_license(config.license_hashes[hash_sum],
+                    config.license_translations,
+                    config.license_blacklist)
     else:
         if not config.license_show:
             return
-        print("Unknown license {0} with hash {1}".format(copying, hash_sum))
-        hashUrl = config.license_show % {'HASH': hash_sum}
-        print("Visit {0} to enter".format(hashUrl))
+        util.print_warning("Unknown license {0} with hash {1}".format(copying, hash_sum))
+        hash_url = config.license_show % {'HASH': hash_sum}
+        util.print_warning("Visit {0} to enter".format(hash_url))
 
 
-def license_from_doc(doc):
-    """ Scan for documentation license in the given file """
-    with open(doc, 'r', encoding="latin-1") as content_file:
-        content = content_file.read()
-    if content.find("GNU Free Documentation License, Version 1.3") >= 0:
-        add_license("GFDL-1.3")
-    elif content.find("GNU Free Documentation License, Version 1.2") >= 0:
-        add_license("GFDL-1.2")
-    elif content.find("GNU Free Documentation License, Version 1.1") >= 0:
-        add_license("GFDL-1.1")
-    elif content.find("(The MIT License)") >= 0:
-        add_license("MIT")
+def skip_license(license_path, config):
+    """Check if a given license file path should be skipped."""
+    skip_name = False
+    for skip in config.license_skips:
+        # handle the common tempfile prefix and normalize for
+        # skip lines without a starting '/'
+        skip = skip if skip[0] != '' else skip[1:]
+        skip_path = ['', 'tmp', '*'] + skip
+        if util.globlike_match(license_path, skip_path):
+            util.print_warning(f"Skip license detected for file at {license_path}")
+            skip_name = True
+            break
+    return skip_name
 
 
-#
-# Scan the project directory for things we can use to guess a description
-# and summary
-#
-
-def scan_for_licenses(package, dir):
-    for dirpath, dirnames, files in os.walk(dir):
+def scan_for_licenses(srcdir, config, pkg_name):
+    """Scan the project directory for things we can use to guess a description and summary."""
+    targets = ["copyright",
+               "copyright.txt",
+               "apache-2.0",
+               "artistic.txt",
+               "libcurllicense",
+               "gpl.txt",
+               "gpl2.txt",
+               "gplv2.txt",
+               "notice",
+               "copyrights",
+               "about_bsd.txt"]
+    # look for files that start with copying or licen[cs]e (but are
+    # not likely scripts) or end with licen[cs]e
+    target_pat = re.compile(r"^((copying)|(licen[cs]e)|(e[dp]l-v\d+))|(licen[cs]e)(\.(txt|xml))?|(intel simplified software license.*\.txt)$")
+    for dirpath, dirnames, files in os.walk(srcdir):
         for name in files:
-            if (name.lower() in ("copyright", "apache-2.0", "libcurllicense", "gpl.txt", "gplv2.txt", "notice", "copyrights", "about_bsd.txt") or
-               name.lower().startswith("copying") or name.lower().startswith("license") or name.lower().endswith("license") or name.lower().startswith("licence") or name.lower().endswith("licence")):
-                license_from_copying_hash(os.path.join(dirpath, name))
-                license_from_copying_content(os.path.join(dirpath, name))
-            else:
-                if "." not in name:
-                    continue
-                ext = name.split(".")[-1].lower()
-                if ext in ("man", "texi", "rdoc"):
-                    license_from_doc(os.path.join(dirpath, name))
+            if name.lower() in targets or target_pat.search(name.lower()):
+                license_path = os.path.join(dirpath, name)
+                if not skip_license(license_path, config):
+                    license_from_copying_hash(license_path, srcdir, config, pkg_name)
+            # Also search for license texts in project trees that are
+            # REUSE-compliant, or are in process of adopting this standard (for
+            # example, KDE ecosystem packages). See https://reuse.software for
+            # details. At a basic level, this layout requires a toplevel
+            # `LICENSES` directory that includes separate files (with .txt
+            # extension) for each license text that covers source code, data,
+            # etc elsewhere in the project tree. A variant layout is currently
+            # seen in the DPDK 20.11.3 tree, where the `LICENSES` directory is
+            # named `license` instead.
+            dirbase = os.path.basename(dirpath)
+            if re.search(r'^(LICENSES|licenses?|licensing)$', dirbase) and re.search(r'\.txt$', name):
+                license_path = os.path.join(dirpath, name)
+                if not skip_license(license_path, config):
+                    license_from_copying_hash(license_path, srcdir, config, pkg_name)
 
-    if len(licenses) == 0:
-        print("\n*ERROR: cannot find any license or " + tarball.name + ".license file!\n")
-        quit()
-    else:
-        print("Licenses    : ", " ".join(sorted(licenses)))
+    if not licenses:
+        util.print_fatal(" Cannot find any license or a valid {}.license file!\n".format(pkg_name))
+        sys.exit(1)
+
+    print("Licenses    : ", " ".join(sorted(licenses)))
 
 
-def write_license(file):
-    if len(licenses) == 0:
-        file.write("License  : " + default_license + "\n")
-    else:
-        file.write("License  : " + " ".join(sorted(licenses)))
-        file.write("\n")
+def load_specfile(specfile):
+    """Get licenses from the specfile content."""
+    specfile.licenses = licenses if licenses else [default_license]
+    specfile.license_files = sorted(license_files)
+    specfile.hashes = hashes

autospec/license_blacklist

@@ -0,0 +1,204 @@
+# This file contains the list of strings in the license blacklist
+# in the form: <blacklisted string>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+# For strings that start with '#', escape the '#' as '\#'.
+%
+%license
+(LGPL)
+(new)
+(specified
+*
++
+-
+-MIT
+-or-
+.git
+.md
+.mit
+.txt
+/
+1-2-3
+1.0
+1.1
+2.0
+2BSD
+3-Clause
+3-clause
+3BSD
+=========================
+>=-2
+@CPACK_RPM_PACKAGE_LICENSE@
+AGPLv3+
+ALv2
+AND
+APL-2.0
+APPLICATION__TYPE
+ASL
+Artistic
+BSD
+BSD(2
+BSD(3
+BSD-2-Clause-Views
+BSD-3
+BSD-compatible
+BSD-derived(Repoze)
+BSD-like
+BSD-style
+BSDL
+BSD_3_clause
+BSDish
+BSL
+CC-BY
+Clause
+Commercial
+Corp.
+Distribution
+Domain
+Dual
+Dual GPL/BSD/CPL
+EPL
+Eclipse
+Expat(MIT/X11)
+Expat/MIT
+FOUNDATION
+Foundation
+FreeBSD
+GENERAL
+GFDL
+GNU
+GPL
+GPL+
+GPL-1.0-only
+GPL/BSD
+GPL/BSD/CPL
+GPLv2.1
+General
+IBM
+Jupyter
+LESSER
+LGPL
+LGPL+BSD
+LGPL/MIT
+LICENCE
+Lesser
+Library
+Licences
+License
+License(2.0)
+License(==-2.0)
+License(>=-2)
+License(>=-2.0)
+License(LGPL)
+License(MIT)
+License,
+License-2
+License-2.0
+License-2.0(MPL-2.0)
+License-3(GPLv3)
+Licensing
+Lucent
+MIT-0
+MIT/Expat
+MPL
+Minpack
+Modified
+Mozilla
+Muddy-MIT
+N/A
+New
+OFL
+OR
+Open
+PIL
+PSF-2+
+PUBLIC
+PYTHON
+Permission
+Public
+Revised
+SIL
+SIP
+SOFTWARE
+See
+Set
+Software
+Source
+Standard
+The
+This
+Two-clause
+UN
+UNKNOWN
+Unknown
+Unkown
+Unlimited
+VERSION-2
+Version
+Version-2.0
+Version-3
+WITH
+What
+\#
+a
+advertising
+and
+any
+bsd
+charge
+classifiers)
+clause)
+copy
+cryptsetup-OpenSSL-exception
+details
+domain
+domain.
+dual
+exceptions
+for
+free
+gpl
+granted
+hereby
+http://nmap.org/man/man-legal.html
+into
+is
+it
+later
+later(LGPLv2+)
+license
+licensing
+ndg/httpsclient/LICENCE
+new
+none
+obtaining
+of
+on
+open_source
+option
+option)
+or
+or(at
+others
+person
+public
+released
+software
+style
+terms
+the
+to
+under?
+unencumbered
+unknown
+unrestricted
+uses
+using
+v1.0
+version
+version-2
+version-2.1
+with
+your
+|
+~

autospec/license_hashes

@@ -0,0 +1,288 @@
+# This file contains license hashes and the licenses they are associated with
+# in the form: <hash>, <license>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+0005480dce93b70f7d62fa311f49e3c6c1a6dcfa, MIT
+003da9a1871de68693740b39bee3ddca66707ad3, LGPL-2.1
+00bf1929edfbe209d33d4b58a47e0178280bd606, MIT
+01a6b4bf79aca9b556822601186afab86e8c4fbf, LGPL-2.1
+0292928b4571cd06a52307940df8bcffc4c51ea4, MIT
+0305317c0f694ba11e8f059938fd0c880356e7bc, BSD-3-Clause
+061048a4b794b6b63e2b81f93404a9eaf5f61f11, GPL  Ghostscript
+06877624ea5c77efe3b7e39b0f909eda6e25a4ec, GPL-2.0
+075d599585584bb0e4b526f5c40cb6b17e0da35a, GPL-2.0
+090586b9e4c51fd5ef3c39f25d2469a8be8e33c9, AFL-2.1 GPL-2.0+
+098e171b1c9c7f15e05e696a187328e3c9aa0099, BSD-4-Clause
+0b184ad51ba2a79e85d2288d5fcf8a1ea0481ea4, GPL-2.0
+0e8e850b0580fbaaa0872326cb1b8ad6adda9b0d, LGPL-2.0
+10782dd732f42f49918c839e8a5e2894c508b079, GPL-2.0
+11464e106e37066a94a23fe4912581799daa3e41, 3BSD
+11c746c0c250304a17169f4d6aa4a82d09fd7157, GPL-2.0
+12549569098cbb0efa8a4aa91f5f38068791fe42, GPL-2.0
+13d2034b5ee3cb8d1a076370cf8f0e344a5d0855, GPL-2.0
+14c1f9f1263a82d58615c51369e1443c3d9f15bf, LGPL-2.1
+14daa45932568feb416c5dd45f1cd909294fb3b0, MIT
+14e8b95e4871fb8a09ebbb5082d34a3c1c7da5a3, BSD MPL
+1665590806e57d72419af943374652cebc42571f, LGPL-2.1 MPL1.0
+174c509aa1452ab3a56faa01aec149015c391e99, LGPL-2.1+ GPL-2.0+
+175e59be229a5bedc6be93e958a970385bb04a62, Apache-2.0
+18b0624b094dd81f099af8672ff5a05259249059, GPL-2.0+ LGPL-2.1+
+196f0bcaf72257201b7e9d499a3c5b5f3f75c170, MIT
+1adfe59a1df9ae50ec572004dd5ebd5f45cc85f7, GPL-3.0+
+1b07b3f8b3b7d79c074d00dfb39a5be425e70cb8, GPL-2.0
+1c0c6888759a63c32bca7eb63353af2cd9bd5d9e, bzip2-1.0.6
+1c74b6200eb29fa8c10588f6e32f3f7cf8eda03f, MIT
+1cc9ab3464ce507485f4982670d12916ec5a3c0e, MIT
+1cf6c2c767c8f1d457c8156aafc9220dd9dedbe5, BSD
+1d6467e08284270291d138c7ea07f24689483068, BSD
+1e2bf763a406378034f0722de926e819c31ed211, GPL-3.0
+1f9b387f1c4c2857808696c5f2d7f9f31c26b911, OldMIT
+21e40770dfa802576ee515345cef47ecedff6524, LGPL-2.0+
+235350d09d078e0fb82939483642d7567b1d1245, MIT
+23a1e6369b12379f07c61bbbaa73e13704ebc30e, MIT
+24790001231306234389a547f69e43601471db75, BSD-3-Clause
+2504e714566874eaff0a4ff3ca44d354b135efe9, MIT
+253c30cd74e4812f13e9e561cb54cbab26bc19dc, GPL-2.0
+254aec9ad4aa42eb4f11c7a58ea1d297e14c43ae, MIT
+254f719409a77168a80aed51593017f6f44165c7, LGPL-2.1 MPL-1.1
+2807f3f1c4cb33b214defc4c7ab72f7e4e70a305, MIT
+2a18f0964f8f7f91a74026ffe45927c83441f333, 3BSD
+2a4fd3610e85e2efdefe2dc331b555794f9e0cb3, GPL-3.0
+2aa8684f0abf57011bd8bf5c30b0390cc2c85a03, MIT
+2aaa700763cac84c6ee8ce491a4148064850a761, Chinese something
+2b61cd7d9b22e98804387e896a3cfa382c1bc4ef, MIT
+2b8b815229aa8a61e483fb4ba0588b8b6c491890, Apache-2.0
+2b9d60c2972b476384af9900276837ac81954e80, GPL-2.0
+2c938b2fe439b3d78ca579e5fc5cf49b3218c0ab, MIT
+2cf9061e85ff400986a10c505a6177ff2b5e992d, MIT
+2d29c273fda30310211bbf6a24127d589be09b6c, GPL-2.0
+313e4788e0730c747b41fad603f0fba3765a022d, MIT
+31b3a6df67bb414219baa57025a5c8d6b79555a5, MIT
+329173a86dabff7ed1c8bd39117d410401c46a05, GPL-2.0+ LGPL-2.0+
+32b0302709046c7d107587b2a4435e91bcea2be8, MIT
+3348e5430ba4fb49fa8eb6e9caf4f06266639d0d, EPL-1.0
+3785dd882865259e99bf41ce033b122370e25ce1, MIT
+38abaf3c887f1799769b604e6d1ada4fcb81f8a0, MIT
+38f27835622344a460610effe7888c9c2ef40c74, MPL-1.0 LGPL-2.1
+39a21f33cadea18adcc23bf808d7d5ea6419c8b1, LGPL-2.1
+3b90aaf730fa20460f8fe3fd20c16daf3acaba59, MIT
+3c33e98325d414f0290c82bc32b0d8c28cbeeb9b, MIT
+3c53de6040bbb05b7ad825a48f8d7341f9e3d6f4, MIT
+3ee0019d4f4ea0a9d3f50800833f30dc14e2968e, GPL-2.0
+40a59387a5d3ab7449e77079f542f86018c16bff, GPL-2.0
+40f6d7c0dba74ddd10663dfa50c2659cbe251423, MIT
+430bd4bbe476b94eb686e5751395a242fc2354e0, BSD
+4386a89a4295085459ef1d54f3ab8859f1f1d28c, LGPL-2.1
+4406aeb00f12a2c9cfba8066eef1a7a0c9b32f43, 3BSD
+44f7289042b71631acac29b2f143330d2da2479e, LGPL-2.0
+46243828d8d29354fd01836ba3a12254c072eedc, GFDL-1.2
+47ca90dc423091f8db250f2cc45f28aba88d11ed, 2BSD
+480ff2fd1236e94295d10551b0b333296f639300, LGPL-2.1+
+48703751ffc7d48dc3aa9609be19dd542cf9776b, GPL-3.0
+4882eb6580148a083a7745785ef76768de52086c, MIT
+48d3aad525d9acd423ac6021c44fa4d15d4ee9ad, BSD-3-Clause
+49113556cdf35023603d71001aa073eadbd26b7e, BSD-3-Clause
+4aa399e286b0bfbfba4819711a9c143b79aa01d0, GPL-2.0
+4c7d9cdbd787a28d9d0d4e1a425cd1e826621e41, LGPL-2.1
+4cc77b90af91e615a64ae04893fdffa7939db84c, GPL-2.0
+4cc8868f0a2c423cc8d63fede50ae0fecd58a720, GPL-3.0+
+4d1d37f306ed270cda5b2741fac3abf0a7b012e5, GPL-2.0
+4d328180e5349a9f65193ed4f424300d0c2c6adc, GPL-2.0
+4df5d4b947cf4e63e675729dd3f168ba844483c7, LGPL-2.1
+4e791d47822f12167fcddffaa7b8231b0c3b300a, MIT
+4f35a76dbcc982e002982b97c5d93f3d95c5e57f, MIT
+4f485ab7059ac53d9e3818278ad82217ce976a36, GFDL-1.1
+4ff4760dfc54975ddb40a3455633574849bc5611, BSD-2-Clause
+50147984b983a89bb0a981968bad2e431f0a1423, MIT
+503df7650052cf38efde55e85f0fe363e59b9739, GPL-2.0
+5050f00b9e4605253a0a9d03d227bf0fb74fd676, MIT
+51ae38c0bc1c4f8b798d3181a3673ceebc93c54b, GPL-2.0
+5292f678d9f514464de238e26705c4e81c90366c, MIT Apache-2.0 BSD-3-Clause
+53afb4c86c46b4980f3f8bb129628c55e740226e, LGPL-2.0
+5405311284eab5ab51113f87c9bfac435c695bb9, GPL-2.0
+545f380fb332eb41236596500913ff8d582e3ead, LGPL-2.1
+570d185ea721e7d6aee7426be1b10a800af98aa8, LGPL-2.0
+57e93c186a48f2ccdd65f1872314534524825e94, MIT
+58e79a1175dc5c95f8ac7dc1d23f1ffc61a9fee7, WEIRD
+597bf5f9c0904bd6c48ac3a3527685818d11246d, LGPL-2.1
+5b550d122aa18a381e29e5be5f4814aeb7b6b800, MIT
+5b811940f3ef0c79e86f6ab11b0e4eced5e7eb14, BSD-3-Clause
+5bb165c9b21503806d60c4559af198f569dc46f6, LGPL-2.1
+5c96c2383498d529b9f62385b9db5ed6f290a890, MIT
+5d1a41800996f1c023b69f4b1c3a93b194a66e80, MIT
+5e7b36dfb18c7b8002bb9c41f87b65d280abd2ae, GFDL1.1
+5eb1e4cf540bfae563680ce53c71c261abb5f076, CERT weird
+5fb122a984b09d5c687513bb34a51eeeff2b13a7, BSD-3-Clause GPL-1.0
+5fb362ef1680e635fe5fb212b55eef4db9ead48f, LGPL-2.0
+61200da0f665a7b331f2f17b065b5a948fad40b8, MIT
+613732109bb85dacf28dccf37b7ec0c212124ecd, MIT
+618c9d05efe2932bc3aa4745ecfb548303dd8f08, MIT
+61bb7a8ea669080cfc9e7dbf37079eae70b535fb, MIT
+62066b13a35774834f104c2ff075ce679a0f75a3, EvolutionMix
+62ba6d3ef166a2b04cf4dcec7039c7afa9dcc2bc, MIT
+63b588642ebe669d50ab658ea1d079bb99086396, GPL-2.0
+654d5ed6dd2d6ab7904d4047cde6345730f9d174, GPL-3.0
+6596e03f9467e3d698993a49289a502c4baf7e08, MIT
+669dfab6308e814f23c524fd51f565a39536ec21, MIT
+66c77efd1cf9c70d4f982ea59487b2eeb6338e26, LGPL-2.1
+6700131aba37c25eb7f67b0c4071d02df6de75bc, MIT
+6723aef56fa8b33bec08e1477f4393e6707c973c, MIT
+6745330da3e7bde244b20b96a42eae659644e731, BSD-3-Clause
+68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1, GPL-2.0
+6ae662f45f5458787a717d7280b6074d352de549, MIT
+6af05e5d7c8c49b139d589a27cddd8468e65a5a3, MIT
+6b1edb9ab8161370f1ba8686ec730eb2cc0c1a13, MIT
+6c394114eefb35de30b9675a99cc1500c6e574cb, MIT
+6c6ec2b7a40bad98fa4a5b55c3855258e5d2a05e, LGPL-2.1 GPL-2.0
+70c32f13c99ca9bb7d53d5cb35fa8243dfa30e76, MIT
+70e64fe9090c157e441681779e0f31aad34f35cb, GPL-3.0
+715a897b37b8e535c5812e5f4e663c30b5a67115, MIT
+72b40d717fc8757da66964a3a5b565aae69d8b39, GPL-3.0
+72d42b3c12e7765c1539d6741cbc2054c2d11260, GPL-3.0+
+72f59fbac43fa1a7f0607d7fdba747832e626656, 3BSD
+73ccddd0c27b4492f8182c35f0f7c434b998757c, GPL-2.0
+747f912a17321a200ab786514210c32116eb51b4, MIT
+74a8a6531a42e124df07ab5599aad63870fa0bd4, GPL-2.0
+74acefbff6c13e5c81492d7460b82e2b5ec16c1b, LGPL-2.1
+74cb2bfd42807a875d1386f2bae67be20c4e19cc, MIT
+750b9d9cc986bfc80b47c9672c48ca615cac0c87, BSD-3-Clause
+794a893e510ca5c15c9c97a609ce47b0df74fc1a, BSD-2-Clause
+79cd7284f964b96ba6df6c35dee5e8a92b9e9b49, 3BSD GPL-2.0+
+7b7e5d72495f3cd2b0d1307b28b92df4ad68a0a8, BSD-3-Clause
+7cdb2709d5ce063238a7fae991c5d6cbe3bc7d7f, LGPL-2.1Apache2
+7f3f67aef48ead049bebdab307c04c2e03342710, 3BSD
+7f673c53e3a7840abd45c5869b6ab8aeb2d34c10, MIT
+8088b44375ef05202c0fca4e9e82d47591563609, LGPL-2.1
+80a39eb9544a657a0e23f53a15daff8a1d6a0e7d, LGPL-2.1
+8244c6af44d86382af14ef78dc6f430925cb46a7, Perl
+8313e316436df0f02ce407f0b52b77f8195bfae6, PublicDomain
+83b927c3fa44af01d2515ea8575f8d4848cc10ec, GPL-2.0+ LGPL-2.0+
+83ba6546e00f890f3a26a9bedd264084f8527d5e, LGPL-2.0
+842745cb706f8f2126506f544492f7a80dbe29b3, GPL-3.0
+84d1b5be6b879fb3967c937a581292206a68ab45, MIT
+8570a5c766d8b7eda6fa171a7f6674e335b50c5e, MIT
+8624bcdae55baeef00cd11d5dfcfa60f68710a02, GPL-3.0
+87d59feee73204c7334b7e00ae1cec3f3c583474, MIT
+88aacfcae9d42c894660950b01e7229cc2895cad, MIT
+893c940eb56cd6a45620c65c72b9a8c48bd79217, MIT
+89e91cab64960b7c852e3b31da3802029af6ef88, GPL-2.0
+8e57ffebd0ed4417edc22e3f404ea3664d7fed27, MIT
+8edc79a3e927a60794fbccaaa22835f9e1ef5c9a, GPL-3.0+
+900806db6414f1bb309ab7438c0a5bac52eb3c2b, CCSA3.0
+904c53ec5a9edfd5d08fcef5fb8ff9c844aed213, MIT
+909b58c9b803acb8d063ac6b2147e56afc8055f6, GPL-2.0
+922e3ebe82d80cae2f6c89bbb065ae455d0265b6, MIT
+937157040d49ab4bbcf2070ed8f9b6849fdfa442, MIT
+9384394a530b4a93530da89f95e05ac86c4880a0, MIT
+943fb830004a8813177c761482ac5d583ee09351, GPL-3.0
+94e50e74aa533e042a8fb41a0d5b07460cdc3b91, GPL-2.0
+952348c4126a5946c98385d2fe5a4801735f174d, MIT
+96784c94078c65a3af7fe6e26b60448042aea27a, IPL-1.0
+97cf1f1fdaaff217f90d58faa9d1b2c0e387fd96, MIT
+99b5245b4714b9b89e7584bfc88da64e2d315b81, BSD
+9a1929f4700d2407c70b507b3b2aaf6226a9543c, LGPL-2.1
+9dc4c58e46864b14d95a1368a6ebac37aad89d32, BSD-3-Clause
+9e010b11705daec7ebf9c01a32d6f09ee444968d, GPL-2.0+
+a08228c5c15d51b7072c406b0d30d862af339ada, MIT
+a280389ff9a8a9d58c76b406ac8a9b64d679f587, MIT
+a2f64f2a85f5fd34bda8eb713c3aad008adbb589, MIT
+a4777b6caf11b08abc996727d094e2dafc16caaa, MIT
+a5771f371f97e15ece15ad91386b8897cb06a95a, 3BSD
+a6adc13d0c809ab8cb68e6e3b6eb7571bd0e2920, GPL-3.0
+a7a897a4bde987e597c04f16a9c28f6d3f57916d, GPL-2.0
+a7c968e282631e3b29c66bcf4a438fc72b68be27, MIT
+a9fbc5ee6bc3b991c1fee735b96204001a8d64fe, SGI-B
+aa4e6280a7b029d15f5be03d2858e5e4c1655ac8, BSD-2-Clause
+aade406c56b751113933f3b64890fb8d5b489a2c, LGPL-3.0
+aba8d76d0af67d57da3c3c321caa59f3d242386b, MPL-1.1
+adcc6ae0ba0e1e4fae77bd2b98db0d5a1aae2e75, BSD-2-Clause
+b034fd3ba225f77f2b5853bcc65393b50761dfd7, MIT
+b0fbb2162c8af440cb60a412a145a01195644b3b, 3BSD
+b256632dcce76559734ff0a23330d2898b7d3a3b, GPL-2.0
+b307e9607ffa7ab6dc4ab7c019d0e62a627b9532, MIT
+b37df50521fc00c23c1835b592967fcce90cd23b, MIT
+b47456e2c1f38c40346ff00db976a2badf36b5e3, GPL-2.0
+b55b6493b790dc1291794a6b4f04064ce06bf435, MIT
+b6e9e05950ebcd16852fe9795b564a3f5d976223, Unknown
+b72fac76415d8029e2823485e3fb9ddb650b879e, GPL-2.0
+b7c6cc817dedc6e9c0aff3bb5a50cbc1e7f9548f, MIT
+b7e55cb48a9fb891583a221fa909ccd75e2ceaaf, LGPL-2.0
+b972655523ae3fdb35306d830b8426e4af30d29c, GPL-2.0+
+b9c8da2c5e4162fc7e321fda0b8d3d084681e764, 2BSD
+ba8966e2473a9969bdcab3dc82274c817cfd98a1, LGPL-2.0
+ba905468a032e17d551b8185bccecea5807b7638, GPL-3.0
+bb306f5409529d1c19b78d60f705d37e3c470124, MIT
+bb38ed62633b427973e8bcbe95c88fc1e00fd77d, MIT
+bb40dd090e18e858168d53bf519129e990f156dc, MIT
+bb5cc149358d8c36484f012e24ce694f66bd3b12, FONT
+bbdf8e1d71045a407fee797ad27983e3eafa3ee6, MIT
+bc06cbdf781c87d2df2fe385214f936d010dd2a2, 3BSD
+bc252631805cf037048f64fef562f98c2a0bdc9e, 3BSD
+bd67b052cf8bd1c31dbcdbe56955b6b5d22243f7, MIT
+be0b40ce8f9532b75966a20d14af123d3c6b05aa, GPL-2.0
+be90ae29e7e0977910b397571efe56601c123793, MIT
+bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f, LGPL-2.0
+c2e8ebe3551b42d4a868922ec922a8af6452c48b, LGPL-3.0
+c3795529e138884da8f5b836db7e2fe1025526cd, GPL-2.0
+c3de6dd37753714ce85482acd8292525d4515595, GPL-2.0
+c5041828c23a816cc38e662b61032e959cb86893, LGPL-2.1
+c540cc53356ba854b2673e53fabc05b2f579404c, GPL-2.0
+c61905dc64311e8bcee6afc425fa40f917a45131, GFDL1.1
+c89f43c4122e411a59ba705cfcf89a3f88019fcf, 3BSD
+c931aad3017d975b7f20666cde0953234a9efde3, GPL-2.0
+caeb68c46fa36651acf592771d09de7937926bb3, LGPL-2.1
+cb03b5ef1e3d983356601be29e306eebdc6a9257, MIT
+cb9f6ee72658f9cb9fceb8b4a3d3e280a4586f20, LGPL-2.1
+cbd3c98acb262700c79d534692dc08cf05ec7f00, MIT
+cbfdfb0078ac466712aa0a82242b8fc955b0bc00, MIT
+cc1e4801215dbc785fbc3d2b9910b2f093466041, GPL-2.0
+cc4023c25020d429d9f4a4adf1a2461c8f334597, 4BSD
+ce154c6a0d76d90cd20ef010636eb8e98a4f9fe5, MIT
+ce714346889d1becdd3132cf7380c734a985ecf1, GPL-1.0 Perl
+cf0f32a176b91013767b8a4a727a369156c6558c, MIT
+cf3eaf29116a37a7d9ba773e776104c067c8e5fc, MIT
+cfea9b772a9da707b5f092d3c18b4572fa4bd520, LGPL-2.0
+d004d8de0c32dfd3e86ca5f4a4552332bf5bbb67, GPL-2.0 LGPL-3.0 APL-2.0 GFDL-1.3
+d0bee23991ce69abdc36d8711f54982ca2fd615c, MIT
+d129361510b2cbbf83a0c2d3dc0381b76d87ac9a, LGPL-2.1 MPL-1.1
+d13c7d638806dc37e5c9b6bbc39366cd904d04de, MIT
+d1845ff7fcb2cdc1f5713dd18140e89de2ab0204, MIT
+d25eb6311e5d0398a98818545298bc0fa10efab1, GPL-2.0
+d46d26c49b071f6f17913af46023b538bf1768e7, MIT
+d4e89d1a1e7812dae053aa8cb891f452891df932, GPL-2.0
+d6dad56b15f36205dc3c605a3e5a98ccac1275f2, MIT
+d83b6378d06fdf228b1afc0bf97e09b44bbb2e7b, LGPL-2.0
+d86fb3f981d36e1f2890c7aaea6f8bcbb7ad4a27, Muddy-MIT
+d8b8b6fadbff8363d66d5c703971ab0c86696e26, MIT
+dcc7cc194ef7fcd3e5f8e9bc20ddad371ee1d36c, MIT
+de9f614d7db67dd479cfa87b10a57ecc4abc7295, MIT
+deed0c8eb4eb962dc74006326c98f750e08c8e3f, 2BSD 3BSD LGPL
+df734091d6dbc9d20382b4084e68396a483582bd, 2BSD Ruby
+dfac199a7539a404407098a2541b9482279f690d, GPL-2.0
+e00070d4eca0371dd3ca3df4a2472cc2db599bad, GPL-2.0
+e16364503a826791c0588c304c794ee8d210b7bc, GPL-1.0
+e1d31e42d2a477d6def889000aa8ffc251f2354c, GFDL-1.3
+e2f32873edf33dd9f03272bb5aa47f825369b905, MIT
+e3afbcad74dd08819b5de5ad0338203369731f67, GPL-2.0
+e4a57e19916359bf6f6f36847712b1761204feed, BSD-3-Clause
+e4b3a39cf858dd93c1d5766b8bf1b49fbe9a1d67, BSD-3-Clause
+e60c2e780886f95df9c9ee36992b8edabec00bcc, LGPL-2.1
+e7d563f52bf5295e6dba1d67ac23e9f6a160fab9, GPL-3.0
+e895154f04fa9aa52d1742728d984428f26a764b, LIBPNG
+e911adf5641a09f13fdd5d59962ad37da043df79, MIT
+ea754f5e8f103855a5574468c9101889553c917e, MIT
+edce7bb3398021ac27115ab50d8db9efaf3408c7, LGPL-2.1
+ef51e850423393335a21c5069af73674cdf6753f, GPL-2.0
+f0236e5d5e53eb7b386cd3ac80b852814e3112dc, GPL-2.0 LGPL-2.1
+f33b4d7ca167851483857c947ea7484d533d9b05, GFDL CCBYSA
+f35b1294fe252bc8ec243713fe9e214bbe5c6069, MIT
+f45ee1c765646813b442ca58de72e20a64a7ddba, GPL-3.0
+f4c513b3be30f40bb34a6e4d8c4f45a5bf1b6a10, Meta
+f6e0bb551b32440d84c3886a34ddbd3d1aec7b56, Apache2.0
+f7e86954cf5438549ec8948212f2713024119c68, GPL-1.0+ Artistic
+f9c9a2d3495a0766b4cf20d4b90cfe714dab3dc1, MIT
+f9e3dfab1d42d7282b0654e921706147795be7cb, LGPL-2.1 GPL-2.0+
+faa78032a41c2bfe6bfd1df72ccd1bb7ab4f7a10, GPL-2.0
+fbf1ca3836c050241ed2fde6ac3a515313f30e8b, GPL-2.0
+fd02365360c1936fdd069b08d29c16c6b16e4ea4, MIT
+fddb19e45cceea1a6c3a9b16547b824bb768eceb, MIT

autospec/license_translations

@@ -0,0 +1,106 @@
+# This file contains license strings and the actual licenses they are associated
+# with in the form: <license string>: <license>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+# For strings that start with '#', escape the '#' as '\#'.
+2-clause, BSD-2-Clause
+AGPL-3, AGPL-3.0
+APL-2.0, Apache-2.0
+APL2, Apache-2.0
+APL2.0, Apache-2.0
+ASL 2.0, Apache-2.0
+ASL-2, Apache-2.0
+ASL-2.0, Apache-2.0
+Apache 2.0, Apache-2.0
+Apache License 2.0, Apache-2.0
+Apache License, Version 2.0, Apache-2.0
+Apache, Apache-2.0
+Apache-2, Apache-2.0
+Apache2, Apache-2.0
+Apache2.0, Apache-2.0
+Apachev2, Apache-2.0
+Artistic-1.0+GPL-1.0, Artistic-1.0 GPL-1.0
+Artistic/GPL, Artistic-1.0-Perl GPL-1.0-or-later
+Artistic_2, Artistic-2.0
+BSD(3-clause), BSD-3-Clause
+BSD-2-clause, BSD-2-Clause
+BSD-3-clause, BSD-3-Clause
+BSD_2_clause, BSD-2-Clause
+BSD_3_clause, BSD-3-Clause
+Boost, BSL-1.0
+CC0, CC0-1.0
+CPL, CPL-1.0
+Common Public License Version 1.0, CPL-1.0
+Expat, MIT
+GFDL1.1, GFDL-1.1
+GPL (> 2), GPL-3.0+
+GPL(==-2), GPL-2.0
+GPL(>=-2), GPL-2.0+
+GPL(>=-2.0), GPL-2.0+
+GPL(>=-2.1), GPL-2.0
+GPL(>=-3), GPL-3.0
+GPL(>=-3.0), GPL-3.0+
+GPL(>=2), GPL-2.0+
+GPL(>=3), GPL-3.0+
+GPL-2+, GPL-2.0+
+GPL-2, GPL-2.0
+GPL-2.0+, GPL-2.0+
+GPL-2.0+LGPL-2.1, GPL-2.0 LGPL-2.1
+GPL-2.0, GPL-2.0
+GPL-2.0-or-later, GPL-2.0+
+GPL-3+, GPL-3.0
+GPL-3, GPL-3.0
+GPL-3.0+, GPL-3.0+
+GPL-3.0, GPL-3.0
+GPL2, GPL-2.0
+GPL3, GPL-3.0
+GPLV2, GPL-2.0
+GPLV3, GPL-3.0
+GPLv2+, GPL-2.0+
+GPLv2, GPL-2.0
+GPLv3+, GPL-3.0+
+GPLv3, GPL-3.0
+ISCL, ISC
+LGPL(>=-2), LGPL-2.0+
+LGPL(>=-2.1), LGPL-2.1+
+LGPL(>=-3), LGPL-3.0+
+LGPL(>=2), LGPL-2.0+
+LGPL-2, LGPL-2.0
+LGPL-2.0+, LGPL-2.0+
+LGPL-2.1+, LGPL-2.1+
+LGPL-2.1-or-later, LGPL-2.1+
+LGPL-3+, LGPL-3.0+
+LGPL-3, LGPL-3.0
+LGPLv2+, LGPL-2.0+
+LGPLv2, LGPL-2.0
+LGPLv2.1+, LGPL-2.1+
+LGPLv2.1, LGPL-2.1
+LGPLv3+, LGPL-3.0+
+LGPLv3, LGPL-3.0
+MIT/X, MIT
+MPL-2, MPL-2.0
+MPL2, MPL-2.0
+MPLv1.1, MPL-1.1
+MPLv2, MPL-2.0
+MPLv2.0, MPL-2.0
+MPLv2.0,, MPL-2.0
+PSF, Python-2.0
+Perl, Artistic-1.0-Perl
+Python, Python-2.0
+VIM, Vim
+ZLIB, Zlib
+ZPL 2.1, ZPL-2.1
+ZPL, ZPL-2.0
+apache, Apache-2.0
+artistic2, Artistic-2.0
+artistic_2, Artistic-2.0
+gplv3, GPL-3.0
+http://creativecommons.org/licenses/BSD/, BSD-2-Clause
+http://opensource.org/licenses/MIT, MIT
+http://www.apache.org/licenses/LICENSE-2.0, Apache-2.0
+lgpl, LGPL-2.1
+mit, MIT
+perl, Artistic-1.0-Perl
+w3c, W3C
+zlib, Zlib
+zlib/libpng, zlib-acknowledgement

autospec/logcheck.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+#
+# logcheck.py - part of autospec
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import sys
+
+from util import print_fatal, write_out
+
+
+def log_etc(lines):
+    """Return the content of the START/etc ... END/etc section."""
+    etc = []
+    while True:
+        line = next(lines)
+        line = line.strip()
+        if line == 'END/etc':
+            break
+        if line.startswith('+'):
+            continue
+        etc.append(line)
+    return etc
+
+
+def logcheck(pkg_loc):
+    """Try to discover configuration options that were automatically switched off."""
+    log = os.path.join(pkg_loc, 'results', 'build.log')
+    if not os.path.exists(log):
+        print('build log is missing, unable to perform logcheck.')
+        return
+
+    whitelist = []
+    file_dir = os.path.dirname(os.path.abspath(__file__))
+    file_path = os.path.join(file_dir, 'configure_whitelist')
+    with open(file_path, "r") as whitelistf:
+        for line in whitelistf:
+            if line.startswith("#"):
+                continue
+            whitelist.append(line.rstrip())
+
+    blacklist = []
+    file_dir = os.path.dirname(os.path.abspath(__file__))
+    file_path = os.path.join(file_dir, 'configure_blacklist')
+    with open(file_path, "r") as blacklistf:
+        for line in blacklistf:
+            if line.startswith("#"):
+                continue
+            blacklist.append(line.rstrip())
+
+    with open(log, 'r') as logf:
+        lines = logf.readlines()
+
+    pat = re.compile(r"^checking (?:for )?(.*?)\.\.\. no")
+    misses = []
+    iter_lines = iter(lines)
+    for line in iter_lines:
+        if line.strip() == "START/etc":
+            etc = log_etc(iter_lines)
+            if etc:
+                write_log(pkg_loc, "etc_files", etc)
+        match = None
+        m = pat.search(line)
+        if m:
+            match = m.group(1)
+
+        if "none required" in line:
+            match = None
+
+        if "warning: format not a string literal" in line:
+            match = line
+
+        if not match or match in whitelist:
+            continue
+
+        if match in blacklist:
+            print_fatal("Blacklisted configure-miss is forbidden: " + match)
+            misses.append("Blacklisted configure-miss is forbidden: " + match)
+            write_log(pkg_loc, 'configure_misses', misses)
+            sys.exit(1)
+
+        print("Configure miss: " + match)
+        misses.append("Configure miss: " + match)
+
+    if not misses:
+        return
+
+    write_log(pkg_loc, 'configure_misses', misses)
+
+
+def write_log(pkg_loc, fname, content):
+    """Create log file with content."""
+    write_out(os.path.join(pkg_loc, fname), '\n'.join(sorted(content)))

autospec/patches.py

@@ -1,44 +0,0 @@
-#!/bin/true
-#
-# patches.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Parse config files
-#
-
-
-patches = []
-autoreconf = False
-
-
-def write_patch_header(file):
-    counter = 1
-    for p in patches:
-        file.write("Patch" + str(counter) + ": " + p.split()[0] + "\n")
-        counter = counter + 1
-
-
-def apply_patches(file):
-    counter = 1
-    for p in patches:
-        name = p.split(None, 1)[0]
-        if name == p:
-            options = "-p1"
-        else:
-            options = p.split(None, 1)[1]
-        if not p.split()[0].endswith(".nopatch"):
-            file.write("%%patch%d %s\n" % (counter, options))
-        counter = counter + 1

autospec/pkg_integrity.py

@@ -0,0 +1,805 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+import download
+import util
+
+GPG_CLI = False
+DESCRIPTION = "Performs package signature verification for packages signed with\
+gpg."
+USAGE = """
+Verify package signature when public key present in default keyring:
+{fn} --sig package.tar.gz.asc --tar package.tar.gz
+
+Verify package signature when public key is provided as cleartext
+{fn} --sig package.tar.gz.asc --tar package.tar.gz --pubkey package_author\
+.pubkey
+
+Verify package signature when public key is in a keyring different from \
+default keyring
+{fn} --sig package.tar.gs.asc --tar package.tar.gz --gnupghome /opt/pki/gpghome
+
+Verify package signature when public key is provided as a file and keyring is \
+different from default
+{fn} --sig package.tar.gs.asc --tar package.tar.gs --pubkey package_author.\
+pubkey --gnupghome /opt/pki/gpghome
+
+""".format(fn=__file__)
+
+SEPT = "-------------------------------------------------------------------------------"
+KEYID_TRY = ""
+KEYID = ""
+IMPORTED = ""
+EMAIL = ""
+GNUPGCONF = """keyserver keyserver.ubuntu.com"""
+CMD_TIMEOUT = 20
+ENV = os.environ
+INPUT_GETTER_TIMEOUT = 60
+CHUNK_SIZE = 2056
+
+KEY_CACHE_DIR = os.path.expanduser('~/.cache/clr-pkg-key-cache')
+
+
+def update_gpg_conf(proxy_value):
+    """Set GNUPGCONF with http_proxy value."""
+    global GNUPGCONF
+    GNUPGCONF = "{}\nkeyserver-options http-proxy={}".format(GNUPGCONF, proxy_value)
+
+
+if 'http_proxy' in ENV.keys():
+    update_gpg_conf(ENV.get('http_proxy'))
+elif 'HTTP_PROXY' in ENV.keys():
+    update_gpg_conf(ENV.get('HTTP_PROXY'))
+
+
+# CLI interface to gpg command
+class GPGCliStatus(object):
+    """Mock gpgmeerror."""
+
+    def __init__(self, strerror):
+        """Initialize mock GPGCliStatus."""
+        self.strerror = strerror
+
+
+class GPGCli(object):
+    """CLI wrapper for gpg."""
+
+    @staticmethod
+    def exec_cmd(args):
+        """Popen wrapper."""
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        try:
+            out, err = proc.communicate(timeout=CMD_TIMEOUT)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            out, err = proc.communicate()
+        return out, err, proc.returncode
+
+    def __init__(self, pubkey=None, home=None):
+        """Set GPGCli defaults."""
+        _gpghome = home
+        if _gpghome is None:
+            _gpghome = tempfile.mkdtemp(prefix='tmp.gpghome')
+        os.environ['GNUPGHOME'] = _gpghome
+        self.args = ['gpg', '--homedir', _gpghome]
+        util.write_out(os.path.join(_gpghome, 'gpg.conf'), GNUPGCONF)
+        if pubkey is not None:
+            args = self.args + ['--import', pubkey]
+            _, err, code = self.exec_cmd(args)
+            if code == -9:
+                raise Exception('Command {} timeout after {} seconds'.format(' '.join(args), CMD_TIMEOUT))
+            elif code != 0:
+                raise Exception(err.decode('utf-8'))
+        self._home = _gpghome
+
+    def verify(self, _, tarfile, signature):
+        """Validate tarfile with signature."""
+        # Since autospec can only verify one signature for now, extract the
+        # first signature from the detached signature file.
+        sig_name = signature
+        packets = parse_gpg_packets(signature)
+        if len(packets) > 1:
+            # sig file may be ascii-armored, so dearmor it first...
+            args = self.args + ['--dearmor', '--output', '-', signature]
+            output, err, code = self.exec_cmd(args)
+            if code != 0:
+                return GPGCliStatus(f'Failed to convert {signature} to binary format')
+            num_bytes = packets[0].get("length")
+            if not num_bytes:
+                return GPGCliStatus(f'Cannot verify first signature from {signature}')
+            first_sig = output[:num_bytes]
+            with tempfile.NamedTemporaryFile(prefix="newsig-", dir=self._home, delete=False) as new_sig_file:
+                new_sig_file.write(first_sig)
+                sig_name = new_sig_file.name
+        args = self.args + ['--verify', sig_name, tarfile]
+        output, err, code = self.exec_cmd(args)
+        if code == 0:
+            return None
+        elif code == -9:
+            return GPGCliStatus('Command {} timeout after {} seconds'.format(' '.join(args), CMD_TIMEOUT))
+        return GPGCliStatus(err.decode('utf-8'))
+
+    def import_key(self, keyid):
+        """Import signer key."""
+        args = self.args + ['--recv-keys', keyid]
+        output, err, code = self.exec_cmd(args)
+        if code == 0:
+            return None, output
+        elif code == -9:
+            return GPGCliStatus('Import key timeout after {} seconds, make sure keystore is reachable'.format(CMD_TIMEOUT)), None
+        return GPGCliStatus(err.decode('utf-8')), None
+
+    def export_key(self, keyid):
+        """Export signer key with armor."""
+        args = self.args + ['--armor', '--export', keyid]
+        output, err, code = self.exec_cmd(args)
+        if output.decode('utf-8') == '':
+            return GPGCliStatus(err.decode('utf-8')), None
+        if code == 0:
+            return None, output.decode('utf-8')
+        return GPGCliStatus(err.decode('utf-8')), None
+
+    def display_keyinfo(self, keyfile):
+        """Show signer key information."""
+        args = self.args + ['--list-packet', keyfile]
+        lp, err, code = self.exec_cmd(args)
+        if code != 0:
+            return GPGCliStatus(err.decode('utf-8')), None
+
+        # trim list-packet output to 10 lines
+        lp = "--list-packet:\n" + "\n".join(lp.decode("utf-8").split("\n")[:10])
+
+        args = self.args + ['--fingerprint', os.path.basename(keyfile.replace(".pkey", ""))]
+        fp, err, code = self.exec_cmd(args)
+        if code != 0:
+            return GPGCliStatus(err.decode('utf-8')), None
+
+        fp = "--fingerprint:\n" + fp.decode("utf-8")
+        return None, "{}\n\n{}".format(lp, fp)
+
+
+@contextmanager
+def cli_gpg_ctx(pubkey=None):
+    """Return correctly initialized GPGCli."""
+    _gpghome = None
+    try:
+        _gpghome = tempfile.mkdtemp(prefix='tmp.gpghome')
+        yield GPGCli(pubkey, _gpghome)
+    finally:
+        if _gpghome is not None:
+            _ = subprocess.run(["gpgconf", "--homedir", _gpghome, "--kill", "all"],
+                               stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            del os.environ['GNUPGHOME']
+            shutil.rmtree(_gpghome, ignore_errors=True)
+
+
+# Use gpg command line
+def verify_cli(pubkey, tarball, signature):
+    """Validate tarfile with signature."""
+    with cli_gpg_ctx(pubkey) as ctx:
+        return ctx.verify(pubkey, tarball, signature)
+    raise Exception('Verification did not take place using cli')
+
+
+class Verifier(object):
+    """Base validation class."""
+
+    def __init__(self, **kwargs):
+        """Set default values."""
+        self.url = kwargs.get('url', None)
+        self.package_sign_path = kwargs.get('package_sign_path', None)
+        self.config = kwargs.get('config', None)
+        print(SEPT)
+
+    @staticmethod
+    def quit():
+        """Stop verification."""
+        util.print_fatal("Verification required for build (verify_required option set)")
+        print(SEPT)
+        sys.exit(1)
+
+    @staticmethod
+    def calc_sum(filepath, digest_algo):
+        """Use digest_algo to calculate block sum of a file."""
+        block_size = 4096
+        with open(filepath, 'rb') as fp:
+            digest = digest_algo()
+            for block in iter(lambda: fp.read(block_size), b''):
+                digest.update(block)
+            return digest.hexdigest()
+
+    def print_result(self, result, err_msg=''):
+        """Display verification results."""
+        package_name = ''
+        if self.url is not None:
+            package_name = os.path.basename(self.url)
+        if result:
+            msg = "{} verification was successful ({})".format(package_name, EMAIL)
+            util.print_success(msg)
+        else:
+            msg = "{} verification failed {}".format(package_name, err_msg)
+            util.print_error(msg)
+
+    def __del__(self):
+        """Display partition."""
+        print(SEPT)
+
+
+def get_signature_file(package_url, package_path):
+    """Attempt to build signature file URL and download it."""
+    sign_urls = []
+    netloc = urlparse(package_url).netloc
+    if 'samba.org' in netloc:
+        sign_urls.append(package_url + '.asc')
+    elif 'mirrors.kernel.org' in netloc:
+        sign_urls.append(package_url + '.sig')
+    else:
+        iter = (package_url + "." + ext for ext in ("asc", "sig", "sign"))
+        for sign_url in iter:
+            sign_urls.append(sign_url)
+
+    sign_file = None
+    dest = None
+    for url in sign_urls:
+        dest = os.path.join(package_path, os.path.basename(url))
+        sign_file = download.do_curl(url, dest)
+        if sign_file is not None:
+            return sign_file
+
+    return None
+
+
+def compare_keys(newkey, oldkey):
+    """Key comparison to check against key tampering."""
+    if newkey != oldkey:
+        util.print_fatal('Public key has changed:\n'
+                         '            old key: {}\n'
+                         '            new key: {}\n'
+                         'this is a critical security error, quitting...'
+                         .format(oldkey, newkey))
+        sys.exit(1)
+
+
+# sha256sum Verifier
+class ShaSumVerifier(Verifier):
+    """Extend verification for sha sums."""
+
+    def __init__(self, **kwargs):
+        """Add shalen initialization."""
+        Verifier.__init__(self, **kwargs)
+        self.package_path = kwargs.get('package_path', None)
+        self.shalen = kwargs.get('shalen', 256)
+
+    def verify_sum(self, shasum):
+        """Verify sha sum."""
+        util.print_info("Verifying sha{}sum digest".format(self.shalen))
+        if shasum is None:
+            self.print_result(False, err_msg='Verification requires shasum')
+            return None
+        if os.path.exists(self.package_path) is False:
+            self.print_result(False, err_msg='{} not found'.format(self.package_path))
+            return None
+
+        sha_algo = {
+            256: hashlib.sha256
+        }.get(self.shalen, None)
+
+        if sha_algo is None:
+            self.print_result(False, err_msg='sha{} algorithm not found'.format(self.shalen))
+            return None
+
+        digest = self.calc_sum(self.package_path, sha_algo)
+        self.print_result(digest == shasum)
+        return digest == shasum
+
+
+# MD5 Verifier
+class MD5Verifier(Verifier):
+    """Extend verification for MD5."""
+
+    def __init__(self, **kwargs):
+        """Add MD5 initialization."""
+        Verifier.__init__(self, **kwargs)
+        self.package_path = kwargs.get('package_path', None)
+        self.md5_digest = kwargs.get('md5_digest', None)
+
+    def verify_md5(self):
+        """Verify MD5."""
+        util.print_info("Verifying MD5 digest")
+        if self.md5_digest is None:
+            self.print_result(False, err_msg='Verification requires a md5_digest')
+            return None
+        if os.path.exists(self.package_path) is False:
+            self.print_result(False, err_msg='{} not found'.format(self.package_path))
+            return None
+        md5_digest = self.calc_sum(self.package_path, hashlib.md5)
+        self.print_result(md5_digest == self.md5_digest)
+        return md5_digest == self.md5_digest
+
+
+# gnome.org Verifier
+class GnomeOrgVerifier(ShaSumVerifier):
+    """Verify sha sums for gnome.org."""
+
+    def __init__(self, **kwargs):
+        """Initialize with gnome.org url."""
+        kwargs.update({'shalen': 256})
+        self.package_url = kwargs.get('url', None)
+        ShaSumVerifier.__init__(self, **kwargs)
+
+    @staticmethod
+    def fetch_shasum(shasum_url):
+        """Get shasum file from gnome.org."""
+        data = download.do_curl(shasum_url)
+        if data:
+            return data.getvalue().decode('utf-8')
+        else:
+            return None
+
+    @staticmethod
+    def get_shasum(package_url):
+        """Try and get sha url based on package url."""
+        ext_pos = package_url.find(".tar.")
+        if ext_pos < 0:
+            shasum_url = "{}.sha256sum".format(package_url)
+        else:
+            shasum_url = package_url[:ext_pos] + ".sha256sum"
+        shasum = GnomeOrgVerifier.fetch_shasum(shasum_url)
+        if shasum:
+            return shasum
+        return None
+
+    @staticmethod
+    def parse_shasum(package_url, shasum_text):
+        """Parse shasum from sha url file content."""
+        for line in shasum_text.split('\n'):
+            if not line.strip():
+                continue
+            sha, file = [col for col in line.split() if col != '']
+            if os.path.basename(package_url) == file:
+                return sha
+        return None
+
+    def verify(self):
+        """Verify tar file with sha from gnome.org."""
+        if self.package_url is None:
+            self.print_result(False, err_msg='Package URL can not be None for GnomeOrgVerifier')
+            return None
+        shasum = self.get_shasum(self.package_url)
+        if shasum is None:
+            self.print_result(False, err_msg='Unable to find shasum URL for {}'.format(self.package_url))
+            return None
+        shasum = self.parse_shasum(self.package_url, shasum)
+        if shasum is None:
+            self.print_result(False, err_msg='Unable to parse shasum {}'.format(shasum))
+            return None
+        return self.verify_sum(shasum)
+
+
+# download.qt.io Verifier
+class QtIoVerifier(ShaSumVerifier):
+    """Verify sha256 hashes for download.qt.io."""
+
+    def __init__(self, **kwargs):
+        """Initialize with package URL."""
+        kwargs.update({'shalen': 256})
+        self.package_url = kwargs.get('url', None)
+        ShaSumVerifier.__init__(self, **kwargs)
+
+    def fetch_shasum(self):
+        """Fetch sha256 file associated with the package URL."""
+        shasum_url = "{}.sha256".format(self.package_url)
+        data = download.do_curl(shasum_url)
+        if data:
+            return data.getvalue().decode('utf-8')
+        else:
+            return None
+
+    def parse_shasum(self, content):
+        """Parse sha256 file."""
+        basename = os.path.basename(self.package_url)
+        line = content.split('\n')[0]
+        match = re.match(r'([0-9a-f]{64})\s+' + basename, line)
+        if not match:
+            return None
+        return match.group(1)
+
+    def verify(self):
+        """Verify the source file's sha256 hash."""
+        if self.package_url is None:
+            self.print_result(False, err_msg='Package URL is empty')
+            return None
+        content = self.fetch_shasum()
+        if content is None:
+            self.print_result(False, err_msg='Unable to download sha256 for {}'.format(self.package_url))
+            return None
+        shasum = self.parse_shasum(content)
+        if shasum is None:
+            self.print_result(False, err_msg='Unable to parse sha256 for {}'.format(self.package_url))
+            return None
+        return self.verify_sum(shasum)
+
+
+# GPG Verification
+class GPGVerifier(Verifier):
+    """Verify GPG signature."""
+
+    def __init__(self, **kwargs):
+        """Initialize gpg signature validation."""
+        Verifier.__init__(self, **kwargs)
+        self.key_url = kwargs.get('key_url', None)
+        self.package_path = kwargs.get('package_path', None)
+        self.package_check = kwargs.get('package_check', None)
+        self.interactive = kwargs.get('interactive', False)
+        if not self.key_url and self.package_check:
+            # signature exists locally, don't try to download self.url
+            self.key_url = self.url.rstrip('/') + get_file_ext(self.package_check)
+        if not self.package_sign_path:
+            # the key exists (or will exist) at
+            # <package directory>/<key url basename>
+            self.package_sign_path = os.path.join(os.path.dirname(self.package_path),
+                                                  os.path.basename(self.key_url))
+        # pubkey path is the package directory - this is where imports will go
+        self.pubkey_path = os.path.join(os.path.dirname(self.package_path), "{}.pkey")
+
+    def get_sign(self):
+        """Attempt to download gpg signature file."""
+        sign_file = download.do_curl(self.key_url, self.package_sign_path)
+        if sign_file is not None:
+            return True
+        else:
+            msg = "Unable to download file {}"
+            self.print_result(False, msg.format(self.key_url))
+
+    def verify(self, recursion=False):
+        """Verify file using gpg signature."""
+        global KEYID
+        global EMAIL
+        util.print_info("Verifying GPG signature")
+        if os.path.exists(self.package_path) is False:
+            self.print_result(False, err_msg='{} not found'.format(self.package_path))
+            return None
+        if os.path.exists(self.package_sign_path) is False and self.get_sign() is not True:
+            self.print_result(False, err_msg='{} not found'.format(self.package_sign_path))
+            return None
+        if sign_isvalid(self.package_sign_path) is False:
+            self.print_result(False, err_msg='{} is not a GPG signature'.format(self.package_sign_path))
+            try:
+                os.unlink(self.package_sign_path)
+            except Exception:
+                pass
+            return None
+        # valid signature exists at package_sign_path, operate on it now
+        keyid = get_keyid(self.package_sign_path)
+        if keyid in self.config.keyid_blocklist:
+            self.print_result(False, err_msg='KNOWNBADACTOR: {}'.format(keyid))
+            self.quit()
+
+        # default location first
+        pubkey_loc = self.pubkey_path.format(keyid)
+        cache_key = os.path.join(KEY_CACHE_DIR, pubkey_loc)
+        if os.path.exists(cache_key) and not os.path.exists(pubkey_loc):
+            shutil.copyfile(cache_key, pubkey_loc)
+        elif os.path.exists(pubkey_loc) and not os.path.exists(cache_key):
+            shutil.copyfile(pubkey_loc, cache_key)
+        elif not os.path.exists(pubkey_loc):
+            # attempt import the key interactively if set to do so
+            self.print_result(False, 'Public key {} not found'.format(keyid))
+            if not self.interactive or recursion:
+                return None
+            if attempt_key_import(keyid, pubkey_loc):
+                return self.verify(recursion=True)
+            return None
+        # public key exists or is imported, verify
+        EMAIL = get_email(pubkey_loc)
+        sign_status = verify_cli(pubkey_loc, self.package_path, self.package_sign_path)
+        if not sign_status:
+            if self.config.old_keyid:
+                compare_keys(KEYID_TRY, self.config.old_keyid)
+            self.print_result(self.package_path)
+            KEYID = KEYID_TRY
+            self.config.signature = self.key_url
+            self.config.config_opts['verify_required'] = True
+            self.config.rewrite_config_opts()
+            return True
+        else:
+            self.print_result(False, err_msg=sign_status.strerror)
+            self.quit()
+
+
+def quit_verify():
+    """Halt build due to verification being required."""
+    Verifier.quit()
+
+
+VERIFIER_TYPES = {
+    '.gz': GPGVerifier,
+    '.tgz': GPGVerifier,
+    '.tar': GPGVerifier,
+    '.bz2': GPGVerifier,
+    '.xz': GPGVerifier,
+    '.zip': GPGVerifier,
+    '.zst': GPGVerifier,
+}
+
+
+def get_file_ext(filename):
+    """Return filename extension."""
+    return os.path.splitext(filename)[1]
+
+
+def get_verifier(filename):
+    """Return verification based on filename."""
+    ext = get_file_ext(filename)
+    return VERIFIER_TYPES.get(ext, None)
+
+
+def get_input(message, default):
+    """Parse user input."""
+    try:
+        import_key = input(message)
+        if import_key == '':
+            import_key = default
+        return import_key.lower() == 'y'
+    except Exception:
+        return None
+
+
+def input_timeout(signum, frame):
+    """Handle input timeouts."""
+    print('\ninput timed out')
+    raise Exception('keyboard timed out')
+
+
+class InputGetter(object):
+    """Simple class for user input."""
+
+    def __init__(self, message='?', default='N', timeout=INPUT_GETTER_TIMEOUT):
+        """Initialize input class."""
+        self.message = message
+        self.default = default
+        self.timeout = timeout
+        signal.signal(signal.SIGALRM, input_timeout)
+
+    def get_answer(self):
+        """Read user input."""
+        signal.alarm(self.timeout)
+        inpt = get_input(self.message, self.default)
+        signal.alarm(0)
+        return inpt
+
+
+def attempt_key_import(keyid, key_fullpath):
+    """Ask user to import key."""
+    global IMPORTED
+    print(SEPT)
+    ig = InputGetter('\nDo you want to attempt to import keyid {}: (y/N) '.format(keyid))
+    import_key_answer = ig.get_answer()
+    if import_key_answer in [None, False]:
+        return False
+    with cli_gpg_ctx() as ctx:
+        err, _ = ctx.import_key(keyid)
+        if err is not None:
+            util.print_error(err.strerror)
+            return False
+        err, key_content = ctx.export_key(keyid)
+        if err is not None:
+            util.print_error(err.strerror)
+            return False
+        util.write_out(key_fullpath, key_content)
+        print('\n')
+        util.print_success('Public key id: {} was imported'.format(keyid))
+        err, content = ctx.display_keyinfo(key_fullpath)
+        if err is not None:
+            util.print_error('Unable to parse {}, will be removed'.format(key_fullpath))
+            os.unlink(key_fullpath)
+            return False
+        print("\n", content)
+        ig = InputGetter(message='\nDo you want to keep this key: (Y/n) ', default='y')
+        if ig.get_answer() is True:
+            IMPORTED = content
+            cache_key = os.path.join(KEY_CACHE_DIR, key_fullpath)
+            if not os.path.isfile(cache_key):
+                shutil.copyfile(key_fullpath, cache_key)
+            else:
+                util.print_warning(f"Re-imported key in cache: {cache_key}")
+            return True
+        else:
+            os.unlink(key_fullpath)
+    return False
+
+
+def parse_gpg_packets(filename, verbose=True):
+    """Return a list with metadata about each packet from a GPG key or signature."""
+    args = ["gpg", "--list-packets", filename]
+    try:
+        out, err = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        if err.decode('utf-8') != '' and verbose is True:
+            print(err.decode('utf-8'))
+            return None
+        out = out.decode('utf-8')
+        packets = []
+        packet = {}
+        for line in out.splitlines():
+            complete = False
+            m = re.search(r'^# off=(\d+) ctb=.* tag=.* hlen=(\d+) plen=(\d+).*$', line)
+            if m:
+                packet["offset"] = int(m.group(1))
+                packet["length"] = int(m.group(2)) + int(m.group(3))
+            m = re.search(r'^:(signature) packet:.* keyid ([0-9A-F]+)$', line)
+            if m and "type" not in packet:
+                packet["type"] = m.group(1)
+                packet["keyid"] = m.group(2)
+                complete = True
+            m = re.search(r'^:(user ID) packet: "(.*) <(.+?)>"', line)
+            if m and "type" not in packet:
+                packet["type"] = m.group(1)
+                packet["user"] = m.group(2)
+                packet["email"] = m.group(3)
+                complete = True
+            # add the packet only if we've extracted all data we need from it
+            if complete:
+                packets.append(packet)
+                packet = {}
+        return packets
+    except Exception:
+        return None
+    return None
+
+
+def get_keyid(sig_filename):
+    """Get keyid from GPG pubkey or signature file and set global KEYID_TRY."""
+    global KEYID_TRY
+    keyid = None
+    packets = parse_gpg_packets(sig_filename)
+    if packets:
+        for p in packets:
+            if "keyid" in p:
+                keyid = p["keyid"]
+                break
+    KEYID_TRY = keyid
+    return keyid.upper() if keyid else None
+
+
+def get_email(pubkey):
+    """Get user email address from GPG pubkey file and set global EMAIL."""
+    email = None
+    packets = parse_gpg_packets(pubkey)
+    if packets:
+        for p in packets:
+            if "email" in p:
+                email = p["email"]
+                break
+    return email
+
+
+def sign_isvalid(sig_filename):
+    """Get keyid from signature file."""
+    keyid = None
+    packets = parse_gpg_packets(sig_filename, verbose=False)
+    if packets:
+        keyid = packets[0].get("keyid")
+    return keyid is not None
+
+
+def filename_from_url(url):
+    """Run os.path.basename for a url."""
+    return os.path.basename(url)
+
+
+def apply_verification(verifier, **kwargs):
+    """Attempt to run verification routine."""
+    if verifier is None:
+        util.print_error("Package is not verifiable (yet)")
+    else:
+        v = verifier(**kwargs)
+        return v.verify()
+
+
+def from_disk(url, package_path, package_check, config, interactive=True):
+    """Run verification."""
+    verifier = get_verifier(package_path)
+    return apply_verification(verifier,
+                              **{
+                                  'package_path': package_path,
+                                  'package_check': package_check,
+                                  'url': url,
+                                  'interactive': interactive,
+                                  'config': config,
+                              })
+
+
+def attempt_verification_per_domain(package_path, url):
+    """Use url domain name to set verification type."""
+    netloc = urlparse(url).netloc
+    if 'download.gnome.org' in netloc:
+        domain = 'gnome.org'
+    elif 'download.qt.io' in netloc:
+        domain = 'qt.io'
+    else:
+        domain = 'unknown'
+    verifier = {
+        'gnome.org': GnomeOrgVerifier,
+        'qt.io': QtIoVerifier,
+    }.get(domain, None)
+
+    if verifier is None:
+        util.print_info('Skipping domain verification')
+        return None
+    else:
+        util.print_info('Verification based on domain {}'.format(domain))
+        return apply_verification(verifier, **{
+                                  'package_path': package_path,
+                                  'url': url})
+
+
+def get_integrity_file(package_path):
+    """Get verification filename."""
+    iter = (package_path + "." + ext for ext in ("asc", "sig", "sign", "sha256"))
+    for sign_file in iter:
+        if os.path.isfile(sign_file):
+            return sign_file
+    return None
+
+
+def check(url, config, interactive=True):
+    """Run verification based on tar file url."""
+    package_name = filename_from_url(url)
+    package_path = os.path.join(config.download_path, package_name)
+    package_check = get_integrity_file(package_path)
+    try:
+        interactive = interactive and sys.stdin.isatty()
+    except ValueError:
+        interactive = False
+    print(SEPT)
+    util.print_info('Performing package integrity verification')
+    verified = None
+    if package_check is not None:
+        verified = from_disk(url, package_path, package_check, config, interactive=interactive)
+    if not verified:
+        util.print_info('None of {}.(asc|sig|sign|sha256) is found in {}'.format(package_name, config.download_path))
+        signature_file = get_signature_file(url, config.download_path)
+        if signature_file is not None:
+            verified = from_disk(url, package_path, signature_file, config, interactive=interactive)
+            if verified is None:
+                util.print_info('Unable to find a signature')
+                verified = attempt_verification_per_domain(package_path, url)
+        else:
+            verified = attempt_verification_per_domain(package_path, url)
+
+    if verified is None and config.config_opts['verify_required']:
+        quit_verify()
+    elif verified is None:
+        print(SEPT)
+    return verified
+
+
+def parse_args():
+    """Set args for tarfile verification."""
+    parser = argparse.ArgumentParser(usage=USAGE, description=DESCRIPTION)
+    parser.add_argument('--tar', required=True,
+                        help='tar file to check signature')
+    parser.add_argument('--sig', required=True,
+                        help='Signature file')
+    parser.add_argument('--pubkey', required=False, default=None,
+                        help='Public key to use for signature verification')
+    parser.add_argument('--gnupghome', required=False, default=None,
+                        help='GNUPGHOME')
+    return parser.parse_args()
+
+
+def load_specfile(specfile):
+    """Set key and email in specfile."""
+    specfile.keyid = KEYID
+    specfile.email = EMAIL

autospec/pkg_scan.py

@@ -0,0 +1,50 @@
+#
+# pkg_scan.py - part of autospec
+# Copyright (C) 2017 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import subprocess
+
+import util
+
+
+def get_whatrequires(pkg, yum_conf):
+    """
+    Write list of packages.
+
+    Write packages that require the current package to a file
+    using dnf repoquery what-requires and --recursive commands.
+    """
+    # clean up dnf cache to avoid 'no more mirrors repo' error
+    try:
+        subprocess.check_output(['dnf', '--config', yum_conf,
+                                 '--releasever', 'clear', 'clean', 'all'])
+    except subprocess.CalledProcessError as err:
+        util.print_warning("Unable to clean dnf repo: {}, {}".format(pkg, err))
+        return
+
+    try:
+        out = subprocess.check_output(['dnf', 'repoquery',
+                                       '--config', yum_conf,
+                                       '--releasever', 'clear',
+                                       '--archlist=src', '--recursive', '--queryformat=%{NAME}',
+                                       '--whatrequires', pkg]).decode('utf-8')
+
+    except subprocess.CalledProcessError as err:
+        util.print_warning("dnf repoquery whatrequires for {} failed with: {}".format(pkg, err))
+        return
+
+    util.write_out('whatrequires', '# This file contains recursive sources that '
+                   'require this package\n' + out)

autospec/pypidata.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+import download
+import util
+
+
+def pip_env():
+    """Generate a copy of os.environ appropriate for pip."""
+    env = os.environ.copy()
+    env["PYTHON_KEYRING_BACKEND"] = "keyring.backends.null.Keyring"
+    return env
+
+
+def pkg_search(name):
+    """Query the pypi json API for name and return True if found."""
+    query = f"https://pypi.org/pypi/{name}/json/"
+    resp = download.do_curl(query)
+    if resp is not None:
+        return True
+    else:
+        return False
+
+
+def fixup_pypi_prefix(name):
+    """Try and chop off the 'pypi-' or 'python-' prefix for names."""
+    name = name.lower().replace('-', '_')
+    for prefix in ["pypi_", "python_"]:
+        if name.startswith(prefix):
+            name = name[len(prefix):]
+    return name
+
+
+def get_pypi_name(name, miss=False):
+    """Try and verify the pypi name for a given package name."""
+    # normalize the name for matching as pypi is case insensitve for search
+    name = name.lower().replace('-', '_')
+    # Common case is the name and the pypi name match
+    if pkg_search(name):
+        return name
+    # Maybe we have a prefix
+    name = fixup_pypi_prefix(name)
+    if pkg_search(name):
+        return name
+    # Some cases where search fails (Sphinx)
+    # Just try the name we were given
+    if miss:
+        return ""
+    return name
+
+
+def _print_command_error(cmd, proc):
+    if isinstance(cmd, list):
+        cmd = " ".join(cmd)
+    util.print_error(f"Command `{cmd}` failed:")
+    for line in proc.stderr.decode('utf-8', errors='surrogateescape').splitlines():
+        util.print_error(line)
+
+
+def get_pypi_metadata(name):
+    """Get metadata for a pypi package."""
+    show = []
+    # Create virtenv to do the pip install (needed for pip show)
+    with tempfile.TemporaryDirectory() as tdir:
+        cmd = ["virtualenv", "--no-periodic-update", tdir]
+        proc = subprocess.run(cmd, capture_output=True)
+        if proc.returncode != 0:
+            _print_command_error(cmd, proc)
+            return ""
+        cmd = f"source bin/activate && pip install {name.removeprefix('pypi_')}"
+        proc = subprocess.run(cmd, cwd=tdir, shell=True, capture_output=True,
+                              env=pip_env())
+        if proc.returncode != 0:
+            _print_command_error(cmd, proc)
+            return ""
+        cmd = f"source bin/activate &> /dev/null && pip show {name.removeprefix('pypi_')}"
+        proc = subprocess.run(cmd, cwd=tdir, shell=True, capture_output=True,
+                              env=pip_env())
+        if proc.returncode != 0:
+            _print_command_error(cmd, proc)
+            return ""
+        show = proc.stdout.decode('utf-8', errors='surrogateescape').splitlines()
+    # Parse pip show for relevent information
+    metadata = {}
+    for line in show:
+        if line.startswith("Name: "):
+            # 'Name: pypi-name'
+            # normalize names -> lowercase and dash to underscore
+            metadata["name"] = line.split()[1].lower().replace('-', '_')
+        elif line.startswith("Summary: "):
+            # 'Summary: <description of the package>'
+            try:
+                metadata["summary"] = line.split(maxsplit=1)[1]
+            except IndexError:
+                # No Summary (haven't seen this case before though)
+                metadata["summary"] = ""
+        elif line.startswith("Requires: "):
+            # 'Requires: dep1, dep2
+            # normalize names -> lowercase and dash to underscore
+            try:
+                reqs = [item.strip().lower().replace('-', '_') for item in line.split(maxsplit=1)[1].split(",")]
+            except IndexError:
+                # No Requires
+                reqs = []
+            metadata["requires"] = reqs
+
+    return json.dumps(metadata)
+
+
+def main():
+    """Standalone pypi metadata query entry point."""
+    pkg_name = sys.argv[1]
+    pypi_name = get_pypi_name(pkg_name)
+    if not pypi_name:
+        util.print_fatal(f"Couldn't find {pkg_name} in pypi")
+        sys.exit(1)
+    pypi_metadata = get_pypi_metadata(pypi_name)
+    print(pypi_metadata)
+
+
+if __name__ == '__main__':
+    main()

autospec/qt_modules

@@ -0,0 +1,68 @@
+# This file maps the qmake entry for the QT or QT_PRIVATE variable to the
+# pkgconfig equivalent, to be added to BuildRequires. Please keep sorted.
+# This file is sorted with LC_COLLATE=C
+# Format: <qmake key>, <pkgconfig value>
+3danimation, Qt63DAnimation
+3dcore, Qt63DCore
+3dextras, Qt63DExtras
+3dinput, Qt63DInput
+3dlogic, Qt63DLogic
+3dquick, Qt63DQuick
+3dquickanimation, Qt63DQuickAnimation
+3dquickextras, Qt63DQuickExtras
+3dquickinput, Qt63DQuickInput
+3dquickrender, Qt63DQuickRender
+3dquickscene2d, Qt63DQuickScene2D
+3drender, Qt63DRender
+bluetooth, Qt6Bluetooth
+charts, Qt6Charts
+concurrent, Qt6Concurrent
+core, Qt6Core
+datavisualization, Qt6DataVisualization
+dbus, Qt6DBus
+declarative, Qt6Declarative
+designer, Qt6Designer
+gamepad, Qt6Gamepad
+gui, Qt6Gui
+help, Qt6Help
+location, Qt6Location
+multimedia, Qt6Multimedia
+multimediawidgets, Qt6MultimediaWidgets
+network, Qt6Network
+networkauth, Qt6NetworkAuth
+nfc, Qt6Nfc
+opengl, Qt6OpenGL
+openglextensions, Qt6OpenGLExtensions
+positioning, Qt6Positioning
+printsupport, Qt6PrintSupport
+purchasing, Qt6Purchasing
+qml, Qt6Qml
+qmltest, Qt6QuickTest
+quick, Qt6Quick
+quickcontrols2, Qt6QuickControls2
+quickwidgets, Qt6QuickWidgets
+remoteobjects, Qt6RemoteObjects
+script, Qt6Script
+scripttools, Qt6ScriptTools
+scxml, Qt6Scxml
+sensors, Qt6Sensors
+serialbus, Qt6SerialBus
+serialport, Qt6SerialPort
+sql, Qt6Sql
+svg, Qt6Svg
+testlib, Qt6Test
+texttospeech, Qt6TextToSpeech
+uitools, Qt6UiTools
+waylandclient, Qt6WaylandClient
+waylandcompositor, Qt6WaylandCompositor
+webchannel, Qt6WebChannel
+webengine, Qt6WebEngine
+webenginecore, Qt6WebEngineCore
+webenginewidgets, Qt6WebEngineWidgets
+webkit, Qt6WebKit
+webkitwidgets, Qt6WebKitWidgets
+websockets, Qt6WebSockets
+widgets, Qt6Widgets
+x11extras, Qt6X11Extras
+xml, Qt6Xml
+xmlpatterns, Qt6XmlPatterns

autospec/specdescription.py

@@ -27,240 +27,242 @@
 #
 
 import os
-import license
+import re
+
+import license
+import util
 
-default_group = "Development/Tools"
 default_description = "No detailed description available"
 default_description_score = 0
 default_summary = "No detailed summary available"
 default_summary_score = 0
 
 
-def clean_license_string(str):
-
-    if str.find("same as") >= 0:
+def clean_license_string(lic):
+    """Clean up license string by replacing substrings."""
+    if lic.find("same as") >= 0:
         return ""
-    str = str.replace(" (", "(")
-    str = str.replace(" v2", "-2")
-    str = str.replace(" v3", "-3")
-    str = str.replace(" 2", "-2")
-    str = str.replace(" 3", "-3")
-    str = str.replace(" <", "<")
-    str = str.replace(" >", ">")
-    str = str.replace("= ", "=")
-    str = str.replace("GPL(>=-2)", "GPL-2.0+")
 
-    str = str.replace("Modified", "")
-    str = str.replace("OSI", "")
-    str = str.replace("Approved", "")
-    str = str.replace("Simplified", "")
-    str = str.replace("file", "")
-    str = str.replace("LICENSE", "")
+    reps = [(" (", "("),
+            (" v2", "-2"),
+            (" v3", "-3"),
+            (" 2", "-2"),
+            (" 3", "-3"),
+            (" <", "<"),
+            (" >", ">"),
+            ("= ", "="),
+            ("GPL(>=-2)", "GPL-2.0+"),
+            ("Modified", ""),
+            ("OSI", ""),
+            ("Approved", ""),
+            ("Simplified", ""),
+            ("file", ""),
+            ("LICENSE", "")]
 
-    return str
+    for sub, rep in reps:
+        lic = lic.replace(sub, rep)
+
+    return lic
 
 
-#
-# Parse any existing RPM specfiles
-#
-
-def description_from_spec(specfile):
-    global default_description
+def assign_summary(summary, score):
+    """Assign summary to default_summary if score is greater than default_summary_score."""
     global default_summary
     global default_summary_score
+    if score > default_summary_score:
+        default_summary = summary
+        default_summary_score = score
+
+
+def assign_description(description, score):
+    """Assign description to default_description if score is greater than default_description_score."""
+    global default_description
     global default_description_score
+    if score > default_description_score:
+        default_description = description
+        default_description_score = score
+
+
+def description_from_spec(specfile, translations, blacklist):
+    """Parse any existing RPM specfiles."""
+    try:
+        with util.open_auto(specfile, 'r') as specfd:
+            lines = specfd.readlines()
+    except FileNotFoundError:
+        return
+
     specdesc = ""
-    phase = 0
-    file = open(specfile, "r", encoding="latin-1")
-    for line in file.readlines():
+    section = False
+    for line in lines:
         if line.startswith("#"):
             continue
 
         if line.startswith("%"):
-            phase = 0
+            section = False
 
-        if line.startswith("License:") and line.find("Copyright") < 0 and line.find("see ") < 0 and line.find("(") < 0:
+        excludes = ["Copyright", "see ", "("]
+        if line.startswith("License:") and not any(e in line for e in excludes):
             splits = line.split(":")[1:]
             words = ":".join(splits).strip()
-            if words in license.license_translations:
-                print("Adding license from spec:", words)
-                license.add_license(words)
+            if words in translations:
+                if license.add_license(words, translations, blacklist):
+                    print("Added license from spec:", words)
             else:
                 words = clean_license_string(words).split()
                 for word in words:
-                    if word.find(":") < 0 or word.startswith('http'):
-                        print("Adding license from spec:", word)
-                        license.add_license(word)
+                    if ":" not in word and not word.startswith("@"):
+                        if license.add_license(word, translations, blacklist):
+                            print("Added license from spec:", word)
 
-        if line.startswith("Summary: ") and default_summary_score < 4:
-            default_summary = line[9:]
-            default_summary_score = 4
-
-        if phase == 1:
-            specdesc = specdesc + line
+        if line.startswith("Summary: "):
+            assign_summary(line[9:], 4)
 
+        specdesc += line if section else ""
+        # Check for %description after assigning the line to specdesc so the
+        # %description string is not included
         if line.endswith("%description\n"):
-            phase = 1
-    if default_description_score < 4:
-        default_description = specdesc
-        default_description_score = 4
-    file.close()
+            section = True
+
+    if len(specdesc) > 10:
+        assign_description(specdesc, 4)
 
 
-def description_from_pkginfo(specfile):
-    global default_description
-    global default_summary
-    global default_summary_score
-    global default_description_score
-    specdesc = ""
-    phase = 0
-    file = open(specfile, "r", encoding="latin-1")
-    for line in file.readlines():
-        if line.find(":") and phase == 1:
-            phase = 0
+def description_from_pkginfo(pkginfo, translations, blacklist):
+    """Parse existing package info files."""
+    try:
+        with util.open_auto(pkginfo, 'r') as pkgfd:
+            lines = pkgfd.readlines()
+    except FileNotFoundError:
+        return
 
-        if line.lower().startswith("license:") and line.find("Copyright") < 0 and line.find("see ") < 0:
+    pkginfo = ""
+    section = False
+    for line in lines:
+        if ":" in line and section:
+            section = False
+
+        excludes = ["Copyright", "see "]
+        if line.lower().startswith("license:") and not any(e in line for e in excludes):
             splits = line.split(":")[1:]
             words = ":".join(splits).strip()
-            if words in license.license_translations:
-                print("Adding license from PKG-INFO:", words)
-                license.add_license(words)
+            if words in translations:
+                if license.add_license(words, translations, blacklist):
+                    print("Added license from PKG-INFO:", words)
             else:
                 words = clean_license_string(words).split()
                 for word in words:
-                    if word.find(":") < 0:
-                        print("Adding license from PKG-INFO:", word)
-                        license.add_license(word)
+                    if ":" not in word:
+                        if license.add_license(word, translations, blacklist):
+                            print("Added license from PKG-INFO:", word)
 
-        if line.startswith("Summary: ") and default_summary_score < 4:
-            default_summary = line[9:]
-            default_summary_score = 4
-
-        if line.startswith("abstract:") and default_summary_score < 4:
-            default_summary = line[9:].strip()
-            default_summary_score = 4
-
-        if phase == 1:
-            specdesc = specdesc + line
+        for sub in ["Summary: ", "abstract: "]:
+            if line.startswith(sub):
+                assign_summary(line[len(sub):].strip(), 4)
 
+        pkginfo += line if section else ""
         if line.startswith("Description:"):
-            phase = 1
-    if default_description_score < 4 and len(specdesc) > 10:
-        default_description = specdesc
-        default_description_score = 4
-    file.close()
+            section = True
 
-#
-# Parse pkgconfig files for Description: lines
-#
+    if len(pkginfo) > 10:
+        assign_description(pkginfo, 4)
 
 
 def summary_from_pkgconfig(pkgfile, package):
-    global default_summary
-    global default_summary_score
-    score = 2
+    """Parse pkgconfig files for Description: lines."""
+    try:
+        with util.open_auto(pkgfile, "r") as pkgfd:
+            lines = pkgfd.readlines()
+    except FileNotFoundError:
+        return
 
-    if pkgfile.find(package + ".pc") >= 0:
-        score = 3
-
-    file = open(pkgfile, "r")
-    for line in file.readlines():
-        if line.startswith("Description:") and default_summary_score < score:
-            default_summary = line[13:]
-            default_summary_score = score
-    file.close()
+    score = 3 if package + ".pc" in pkgfile else 2
+    for line in lines:
+        if line.startswith("Description:"):
+            assign_summary(line[13:], score)
+            # Score will not increase, stop trying
+            break
 
 
-def summary_from_R(pkgfile, package):
-    global default_summary
-    global default_summary_score
-    score = 2
+def summary_from_R(pkgfile):
+    """Parse DESCRIPTION file for Title: lines."""
+    try:
+        with util.open_auto(pkgfile, "r") as pkgfd:
+            lines = pkgfd.readlines()
+    except FileNotFoundError:
+        return
 
-    if pkgfile.find("DESCRIPTION") >= 0:
-        score = 3
-
-    file = open(pkgfile, "r", encoding="latin-1")
-    for line in file.readlines():
-        if line.startswith("Title:") and default_summary_score < score:
-            default_summary = line[7:]
-            default_summary_score = score
-    file.close()
+    for line in lines:
+        if line.startswith("Title:"):
+            assign_summary(line[7:], 3)
+            # Score will not increase, stop trying
+            break
 
 
-#
-# some lines from a readme are just boilerplate and should be skipped
-#
 def skipline(line):
-    if line.find("Copyright") >= 0:
-        return 1
-    if line.find("Free Software Foundation, Inc.") >= 0:
-        return 1
-    if line.find("Copying and distribution of") >= 0:
-        return 1
-    if line.find("are permitted in any") >= 0:
-        return 1
-    if line.find("notice and this notice") >= 0:
-        return 1
-    if line.find("README") >= 0:
-        return 1
-    if line.find("-*-") >= 0:
-        return 1
-
+    """Skip boilerplate readme lines."""
     if line.endswith("introduction"):
-        return 1
-    return 0
+        return True
+
+    skips = ["Copyright",
+             "Free Software Foundation, Inc.",
+             "Copying and distribution of",
+             "are permitted in any",
+             "notice and this notice",
+             "README",
+             "-*-"]
+    return any(s in line for s in skips)
 
 
-#
-# Try to pick the first paragraph or two from the readme file
-#
 def description_from_readme(readmefile):
-    global default_description
-    global default_description_score
-    state = 0
+    """Try to pick the first paragraph or two from the readme file."""
+    try:
+        with util.open_auto(readmefile, "r") as readmefd:
+            lines = readmefd.readlines()
+    except FileNotFoundError:
+        return
+
+    section = False
     desc = ""
-    score = 1
-
-    if readmefile.lower().endswith("readme"):
-        score = 1.5
-
-    file = open(readmefile, "r", encoding="latin-1")
-    for line in file.readlines():
-        if state == 1 and len(line) < 2 and len(desc) > 80:
-            state = 2
-        if state == 0 and len(line) > 2:
-            state = 1
-        if state == 1:
+    for line in lines:
+        if section and len(line) < 2 and len(desc) > 80:
+            # If we are in a section and encounter a new line, break as long as
+            # we already have a description > 80 characters.
+            break
+        if not section and len(line) > 2:
+            # Found the first paragraph hopefully
+            section = True
+        if section:
+            # Copy all non-empty lines into the description
             if skipline(line) == 0 and len(line) > 2:
                 desc = desc + line.strip() + "\n"
 
-    if default_description_score < score:
-        default_description = desc
-        default_description_score = score
-    file.close()
-
-#
-# Scan the project directory for things we can use to guess a description
-# and summary
-#
+    score = 1.5 if readmefile.lower().endswith("readme") else 1
+    assign_description(desc, score)
 
 
-def scan_for_description(package, dir):
-    global default_summary
-    for dirpath, dirnames, files in os.walk(dir):
+def scan_for_description(package, dirn, translations, blacklist):
+    """Scan the project directory for things we can use to guess a description and summary."""
+    test_pat = re.compile(r"tests?")
+    dirpath_seen = ""
+    for dirpath, dirnames, files in os.walk(dirn):
+        if dirpath_seen != dirpath:
+            dirpath_seen = dirpath
+            dirnames[:] = [d for d in dirnames if not re.match(test_pat, d)]
         for name in files:
+            if name.lower().endswith(".pdf"):
+                continue
             if name.lower().endswith(".spec"):
-                description_from_spec(os.path.join(dirpath, name))
+                description_from_spec(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith("pkg-info"):
-                description_from_pkginfo(os.path.join(dirpath, name))
+                description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith("meta.yml"):
-                description_from_pkginfo(os.path.join(dirpath, name))
+                description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith("description"):
-                description_from_pkginfo(os.path.join(dirpath, name))
+                description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith(".pc"):
                 summary_from_pkgconfig(os.path.join(dirpath, name), package)
             if name.startswith("DESCRIPTION"):
-                summary_from_R(os.path.join(dirpath, name), package)
+                summary_from_R(os.path.join(dirpath, name))
             if name.lower().endswith(".pc.in"):
                 summary_from_pkgconfig(os.path.join(dirpath, name), package)
             if name.lower().startswith("readme"):
@@ -268,22 +270,14 @@ def scan_for_description(package, dir):
 
     print("Summary     :", default_summary.strip())
 
-#  print("Summary: ", default_summary)
-#  print("%description")
-#  print(default_description)
 
-
-def write_summary(file):
-    global default_summary
-    global default_group
-    file.write("Summary  : " + default_summary.strip() + "\n")
-    file.write("Group    : " + default_group.strip() + "\n")
-
-
-def write_description(file, package=""):
-    global default_description
-    if len(package) == 0:
-        file.write("\n%description\n" + default_description.strip() + "\n")
+def load_specfile(specfile, description, summary):
+    """Load specfile with parse results."""
+    if description:
+        specfile.default_desc = "\n".join(description)
     else:
-        file.write("\n%description " + package +
-                   "\n Subpackage " + package + "\n\n")
+        specfile.default_desc = default_description
+    if summary:
+        specfile.default_sum = summary[0]
+    else:
+        specfile.default_sum = default_summary

autospec/tarball.py

@@ -17,281 +17,435 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import build
-import buildpattern
-import buildreq
-import files
-import glob
-import hashlib
+import bz2
+import configparser
 import os
 import re
-import shutil
-import subprocess
-import urllib
-import urllib.request
-from util import call
+import sys
+import tarfile
+import zipfile
 
-name = ""
-rawname = ""
-version = ""
-release = "1"
-url = ""
-path = ""
-tarball_prefix = ""
-gcov_file = ""
+import download
+import zstandard as zstd
+from util import do_regex, get_sha1sum, print_fatal, write_out
 
 
-def get_sha1sum(filename):
-    sh = hashlib.sha1()
-    with open(filename, "rb") as f:
-        sh.update(f.read())
-    return sh.hexdigest()
+class Source():
+    """Holds data and methods for source code or archives management."""
+
+    def __init__(self, url, destination, path, pattern=None):
+        """Set default values for source file."""
+        self.url = url
+        self.destination = destination
+        self.path = path
+        self.pattern = pattern
+        self.type = None
+        self.prefix = None
+        self.subdir = None
+
+        # Extra  compressed archives
+        if not self.destination.startswith(':'):
+            self.set_type()
+            self.set_prefix()
+
+    def set_type(self):
+        """Determine compression type."""
+        if self.url.lower().endswith(('.zip', 'jar')):
+            self.type = 'zip'
+        elif self.url.lower().endswith(('.bz2')) and not self.url.lower().endswith(('.tar.bz2')):
+            self.type = 'bz2'
+        elif self.url.lower().endswith('.zst'):
+            self.type = 'zst'
+        else:
+            self.type = 'tar'
+
+    def set_prefix(self):
+        """Determine the prefix and subdir if no prefix."""
+        prefix_method = getattr(self, 'set_{}_prefix'.format(self.type))
+        prefix_method()
+        # When there is no prefix, create subdir
+        if not self.prefix:
+            self.subdir = os.path.splitext(os.path.basename(self.path))[0]
+
+    def set_tar_prefix(self):
+        """Determine prefix folder name of tar file."""
+        if tarfile.is_tarfile(self.path):
+            with tarfile.open(self.path, 'r') as content:
+                lines = content.getnames()
+                # When tarball is not empty
+                if len(lines) == 0:
+                    print_fatal("Tar file doesn't appear to have any content")
+                    sys.exit(1)
+                elif len(lines) > 1:
+                    if 'package.xml' in lines and self.pattern in ['phpize']:
+                        lines.remove('package.xml')
+                    self.prefix = os.path.commonpath(lines)
+        else:
+            print_fatal("Not a valid tar file.")
+            sys.exit(1)
+
+    def set_zst_prefix(self):
+        """Determine prefix folder name of tar.zst file."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            lines = content.getnames()
+            if len(lines) == 0:
+                print_fatal("Zstd compressed tar file doesn't appear to have any content")
+                sys.exit(1)
+            elif len(lines) > 1:
+                self.prefix = os.path.commonpath(lines)
+
+    def set_bz2_prefix(self):
+        """No prefix for plain bz2 archives."""
+
+    def set_zip_prefix(self):
+        """Determine prefix folder name of zip file."""
+        if zipfile.is_zipfile(self.path):
+            with zipfile.ZipFile(self.path, 'r') as content:
+                lines = content.namelist()
+                # When zipfile is not empty
+                if len(lines) > 0:
+                    self.prefix = os.path.commonpath(lines)
+                else:
+                    print_fatal("Zip file doesn't appear to have any content")
+                    sys.exit(1)
+        else:
+            print_fatal("Not a valid zip file.")
+            sys.exit(1)
+
+    def extract(self, base_path):
+        """Prepare extraction path and call specific extraction method."""
+        if not self.prefix:
+            extraction_path = os.path.join(base_path, self.subdir)
+        else:
+            extraction_path = base_path
+
+        extract_method = getattr(self, 'extract_{}'.format(self.type))
+        try:
+            extract_method(extraction_path)
+        except tarfile.AbsoluteLinkError:
+            pass
+
+    def extract_tar(self, extraction_path):
+        """Extract tar in path."""
+        with tarfile.open(self.path) as content:
+            content.extractall(path=extraction_path, filter='data')
+
+    def extract_bz2(self, extraction_path):
+        """Extract plain bz2 file in path."""
+        with bz2.BZ2File(self.path, 'rb') as content:
+            data = content.read()
+            newfile = self.path.rsplit('/', 1)[1]
+            newfile = newfile.rsplit('.bz2', 1)[0]
+            with open(os.path.join(extraction_path), mode='wb') as f:
+                f.write(data)
+
+    def extract_zip(self, extraction_path):
+        """Extract zip in path."""
+        with zipfile.ZipFile(self.path, 'r') as content:
+            content.extractall(path=extraction_path)
+
+    def extract_zst(self, extraction_path):
+        """Extract zst in path."""
+        with tarfile.open(fileobj=zstd.open(self.path, 'rb'), mode='r|') as content:
+            content.extractall(path=extraction_path, filter='data')
 
 
-def really_download(url, destination, tarfile):
-    with urllib.request.urlopen(url) as response:
-        with open(destination, "wb") as dfile:
-            dfile.write(response.read())
+def convert_version(ver_str, name):
+    """Remove disallowed characters from the version."""
+    # banned substrings. It is better to remove these here instead of filtering
+    # them out with expensive regular expressions
+    banned_subs = ["x86.64", "source", "src", "all", "bin", "release", "rh",
+                   "ga", ".ce", "lcms", "onig", "linux", "gc", "sdk", "orig",
+                   "jurko", "%2f", "%2F", "%20", "x265", "autotools"]
+
+    # package names may be modified in the version string by adding "lib" for
+    # example. Remove these from the name before trying to remove the name from
+    # the version
+    name_mods = ["lib", "core", "pom", "opa-"]
+
+    # enforce lower-case strings to make them easier to standardize
+    ver_str = ver_str.lower()
+    # remove the package name from the version string
+    ver_str = ver_str.replace(name.lower(), '')
+    # handle modified name substrings in the version string
+    for mod in name_mods:
+        ver_str = ver_str.replace(name.replace(mod, ""), "")
+
+    # replace illegal characters
+    ver_str = ver_str.strip().replace('-', '.').replace('_', '.').replace('+', '.')
+
+    # remove banned substrings
+    for sub in banned_subs:
+        ver_str = ver_str.replace(sub, "")
+
+    # remove consecutive '.' characters
+    while ".." in ver_str:
+        ver_str = ver_str.replace("..", ".")
+
+    return ver_str.strip(".")
 
 
-def check_or_get_file(url, tarfile):
-    tarball_path = build.download_path + "/" + tarfile
-    if not os.path.isfile(tarball_path):
-        really_download(url, tarball_path, tarfile)
-    return tarball_path
+class Content():
+    """Detect static information about the project."""
 
+    def __init__(self, url, name, version, archives, config, base_path):
+        """Initialize Default content settings."""
+        self.name = name
+        self.rawname = ""
+        self.version = version
+        self.release = "1"
+        self.url = url
+        self.path = ""
+        self.tarball_prefix = ""
+        self.gcov_file = ""
+        self.archives = archives
+        self.giturl = ""
+        self.repo = ""
+        self.domain = ""
+        self.prefixes = dict()
+        self.config = config
+        self.base_path = base_path
 
-def build_untar(tarball_path):
-    tarball_contents = subprocess.check_output(
-        ["tar", "-tf", tarball_path], universal_newlines=True)
-    extract_cmd = "tar --directory={0} -xf {1}".format(build.base_path, tarball_path)
-    if tarball_contents:
-        tarball_prefix = tarball_contents.splitlines()[0].rsplit("/")[0]
-        if tarball_prefix == ".":
-            tarball_prefix = tarball_contents.splitlines()[0].rsplit("/")[1]
-    return extract_cmd, tarball_prefix
+    def write_upstream(self, sha, tarfile, mode="w"):
+        """Write the upstream hash to the upstream file."""
+        write_out(os.path.join(self.config.download_path, "upstream"),
+                  os.path.join(sha, tarfile) + "\n", mode=mode)
 
+    def extract_sources(self, main_src, archives_src):
+        """Extract sources."""
+        full_list_src = [main_src] + archives_src
+        for src in full_list_src:
+            if src.destination != ':':
+                src.extract(self.base_path)
 
-def download_tarball(url_argument, name_argument, archives, target_dir):
-    global name
-    global rawname
-    global version
-    global url
-    global path
-    global tarball_prefix
-    global gcov_file
+    def check_or_get_file(self, upstream_url, tarfile, mode="w"):
+        """Download tarball from url unless it is present locally."""
+        tarball_path = self.config.download_path + "/" + tarfile
+        if not os.path.isfile(tarball_path):
+            download.do_curl(upstream_url, dest=tarball_path, is_fatal=True)
+            self.write_upstream(get_sha1sum(tarball_path), tarfile, mode)
+        else:
+            self.write_upstream(get_sha1sum(tarball_path), tarfile, mode)
+        return tarball_path
 
-    url = url_argument
-    tarfile = os.path.basename(url)
-    pattern_options = [
-        r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)",
-        r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)",
-        r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar",
-        r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar",
-    ]
-    for pattern in pattern_options:
-        p = re.compile(pattern)
-        m = p.search(tarfile)
-        if m:
-            name = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
-            break
+    def process_main_source(self, url):
+        """Download and get important information from main source code."""
+        src_path = self.check_or_get_file(url, os.path.basename(url))
+        main_src = Source(url, '', src_path, self.config.default_pattern)
+        return main_src
 
-    rawname = name
-    # R package
-    if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0:
-        buildpattern.set_build_pattern("R", 10)
-        files.want_dev_split = 0
-        buildreq.add_buildreq("clr-R-helpers")
-        p = re.compile(r"([A-Za-z0-9]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz")
-        m = p.search(tarfile)
-        if m:
-            name = "R-" + m.group(1).strip()
-            rawname = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
+    def print_header(self):
+        """Print header for autospec run."""
+        print("\n")
+        print("Processing", self.url)
+        print("=" * 105)
+        print("Name        :", self.name)
+        print("Version     :", self.version)
+        print("Prefix      :", self.tarball_prefix)
 
-    if url_argument.find("pypi.python.org") > 0:
-        buildpattern.set_build_pattern("distutils", 10)
+    def set_giturl_and_domain(self):
+        """Set giturl and domain from the config (needs config refactor)."""
+        options_path = os.path.join(self.config.download_path, 'options.conf')
+        if os.path.exists(options_path):
+            config_f = configparser.ConfigParser(interpolation=None)
+            config_f.read(options_path)
+            if "package" in config_f.sections():
+                if "giturl" in config_f["package"]:
+                    self.giturl = config_f["package"].get("giturl")
+                if "domain" in config_f["package"]:
+                    self.domain = config_f["package"].get("domain")
 
-    if url_argument.find(".cpan.org/CPAN/") > 0:
-        buildpattern.set_build_pattern("cpan", 10)
-        if name:
-            name = "perl-" + name
-    if url_argument.find(".metacpan.org/") > 0:
-        buildpattern.set_build_pattern("cpan", 10)
-        if name:
+    def name_and_version(self, filemanager):
+        """Parse the url for the package name and version."""
+        tarfile = os.path.basename(self.url)
+
+        # If both name and version overrides are set via commandline, set the name
+        # and version variables to the overrides and bail. If only one override is
+        # set, continue to auto detect both name and version since the URL parsing
+        # handles both. In this case, wait until the end to perform the override of
+        # the one that was set. An extra conditional, that version_arg is a string
+        # is added to enable a package to have multiple versions at the same time
+        # for some language ecosystems.
+        if self.name and self.version:
+            # rawname == name in this case
+            self.rawname = self.name
+            self.version = convert_version(self.version, self.name)
+            return
+
+        name = self.name
+        self.rawname = self.name
+        version = ""
+        # it is important for the more specific patterns to come first
+        pattern_options = [
+            # handle font packages with names ending in -nnndpi
+            r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
+            r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
+        ]
+        match = do_regex(pattern_options, tarfile)
+        if match:
+            name = match.group(1).strip()
+            version = convert_version(match.group(2), name)
+
+        # R package
+        if "cran.r-project.org" in self.url or "cran.rstudio.com" in self.url and name:
+            filemanager.want_dev_split = False
+            self.rawname = name
+            name = "R-" + name
+
+        if ".cpan.org/" in self.url or ".metacpan.org/" in self.url and name:
             name = "perl-" + name
 
-    if url_argument.find("github.com") > 0:
-        p = re.compile(r"https://github.com/.*/(.*?)/archive/v?(.*).tar")
-        m = p.search(url_argument)
-        if m:
-            name = m.group(1).strip()
-            version = m.group(2).strip()
+        if "github.com" in self.url:
+            # define regex accepted for valid packages, important for specific
+            # patterns to come before general ones
+            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/.*/(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
+                               r"https?://github.com/(.*)/.*-downloads/releases/download/.*?/(.*)-(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/releases/download/(.*)/",
+                               r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar"]
 
-    if url_argument.find("bitbucket.org") > 0:
-        p = re.compile(r"https://bitbucket.org/.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9_.]*).tar")
-        m = p.search(url_argument)
-        if m:
-            name = m.group(1).strip()
-            version = m.group(2).strip().replace('_', '.')
+            match = do_regex(github_patterns, self.url)
+            if match:
+                self.repo = match.group(2).strip()
+                if self.repo not in name:
+                    # Only take the repo name as the package name if it's more descriptive
+                    name = self.repo
+                elif name != self.repo:
+                    name = re.sub(r"release-", '', name)
+                    name = re.sub(r"\d*$", '', name)
+                self.rawname = name
+                version = match.group(3).replace(name, '')
+                if "/archive/" not in self.url:
+                    version = re.sub(r"^[-_.a-zA-Z]+", "", version)
+                version = convert_version(version, name)
+                if not self.giturl:
+                    self.giturl = "https://github.com/" + match.group(1).strip() + "/" + self.repo + ".git"
 
-    # ruby
-    if url_argument.find("rubygems.org/") > 0:
-        buildpattern.set_build_pattern("ruby", 10)
-        p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem")
-        m = p.search(tarfile)
-        if m:
-            buildreq.add_buildreq("ruby")
-            buildreq.add_buildreq("rubygem-rdoc")
-            name = "rubygem-" + m.group(1).strip()
-            rawname = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
+        # SQLite tarballs use 7 digit versions, e.g 3290000 = 3.29.0, 3081002 = 3.8.10.2
+        if "sqlite.org" in self.url:
+            major = version[0]
+            minor = version[1:3].lstrip("0").zfill(1)
+            patch = version[3:5].lstrip("0").zfill(1)
+            build = version[5:7].lstrip("0")
+            version = major + "." + minor + "." + patch + "." + build
+            version = version.strip(".")
 
-    # override from commandline
-    if name_argument and name_argument[0] != name:
-        pattern = name_argument[0] + r"[\-]*(.*)\.(tgz|tar|zip)"
-        p = re.compile(pattern)
-        m = p.search(tarfile)
-        if m:
-            name = name_argument[0]
-            rawname = name
-            version = m.group(1).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
-            if version.startswith('.'):
-                version = version[1:]
-        else:
-            name = name_argument[0]
+        # Construct gitlab giturl for GNOME projects, and update previous giturls
+        # that pointed to the GitHub mirror.
+        if "download.gnome.org" in self.url:
+            if not self.giturl or "github.com/GNOME" in self.giturl or "git.gnome.org" in self.giturl:
+                self.giturl = "https://gitlab.gnome.org/GNOME/{}".format(name)
 
-    if not name:
-        split = url_argument.split('/')
-        if len(split) > 3 and split[-2] in ('archive', 'tarball'):
-            name = split[-3]
-            version = split[-1]
-            if version.startswith('v'):
-                version = version[1:]
-            # remove extension
-            version = '.'.join(version.split('.')[:-1])
-            if version.endswith('.tar'):
-                version = '.'.join(version.split('.')[:-1])
+        if "mirrors.kernel.org" in self.url:
+            m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", self.url)
+            if m:
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    b = version.find("-")
-    if b >= 0:
-        b = b + 1
-        version = version[b:]
+        if "sourceforge.net" in self.url:
+            scf_pats = [r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz",
+                        r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz"]
+            match = do_regex(scf_pats, self.url)
+            if match:
+                name = match.group(1).strip()
+                version = convert_version(match.group(2), name)
 
-    if version[0] in ['v', 'r']:
-        version = version[1:]
+        if "bitbucket.org" in self.url:
+            bitbucket_pats = [r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)",
+                              r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)"]
 
-    assert name != ""
+            match = do_regex(bitbucket_pats, self.url)
+            if match:
+                name = match.group(1).strip()
+                version = convert_version(match.group(2), name)
 
-    if not target_dir:
-        build.download_path = os.getcwd() + "/" + name
-    else:
-        build.download_path = target_dir
-    call("mkdir -p %s" % build.download_path)
+        if "gitlab.com" in self.url:
+            # https://gitlab.com/leanlabsio/kanban/-/archive/1.7.1/kanban-1.7.1.tar.gz
+            m = re.search(r"gitlab\.com/.*/(.*)/-/archive/(?:VERSION_|[vVrR])?(.*)/", self.url)
+            if m:
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    gcov_path = build.download_path + "/" + name + ".gcov"
-    if os.path.isfile(gcov_path):
-        gcov_file = name + ".gcov"
+        if "git.sr.ht" in self.url:
+            # https://git.sr.ht/~sircmpwn/scdoc/archive/1.9.4.tar.gz
+            m = re.search(r"git\.sr\.ht/.*/(.*)/archive/(.*).tar.gz", self.url)
+            if m:
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    tarball_path = check_or_get_file(url, tarfile)
-    sha1 = get_sha1sum(tarball_path)
-    with open(build.download_path + "/upstream", "w") as file:
-        file.write(sha1 + "/" + tarfile + "\n")
+        if "pigeonhole.dovecot.org" in self.url:
+            # https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3-pigeonhole-0.5.20.tar.gz
+            if m := re.search(r"pigeonhole\.dovecot\.org/releases/.*/dovecot-[\d\.]+-(\w+)-([\d\.]+)\.[^\d]", self.url):
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    tarball_prefix = name + "-" + version
-    if tarfile.lower().endswith('.zip'):
-        tarball_contents = subprocess.check_output(
-            ["unzip", "-l", tarball_path], universal_newlines=True)
-        if tarball_contents and len(tarball_contents.splitlines()) > 3:
-            tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1]
-        extract_cmd = "unzip -d {0} {1}".format(build.base_path, tarball_path)
+        if ".ezix.org" in self.url:
+            # https://www.ezix.org/software/files/lshw-B.02.19.2.tar.gz
+            if m := re.search(r"(\w+)-[A-Z]\.(\d+(?:\.\d+)+)", self.url):
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    elif tarfile.lower().endswith('.gem'):
-        tarball_contents = subprocess.check_output(
-            ["gem", "unpack", "--verbose", tarball_path], universal_newlines=True)
-        extract_cmd = "gem unpack --target={0} {1}".format(build.base_path, tarball_path)
-        if tarball_contents:
-            tarball_prefix = tarball_contents.splitlines()[-1].rsplit("/")[-1]
-            if tarball_prefix.endswith("'"):
-                tarball_prefix = tarball_prefix[:-1]
-    else:
-        extract_cmd, tarball_prefix = build_untar(tarball_path)
+        if self.name and not version:
+            # In cases where we have a name but no version
+            # use what is after the name.
+            # https://invisible-mirror.net/archives/lynx/tarballs/lynx2.8.9rel.1.tar.gz
+            postname = tarfile.split(self.name)[-1]
+            no_extension = os.path.splitext(postname)[0]
+            if no_extension.endswith('.tar'):
+                no_extension = os.path.splitext(no_extension)[0]
+            version = convert_version(no_extension, self.name)
 
-    print("\n")
+        # override name and version from commandline
+        self.name = self.name if self.name else name
+        self.version = self.version if self.version else version
 
-    print("Processing", url_argument)
-    print(
-        "=============================================================================================")
-    print("Name        :", name)
-    print("Version     :", version)
-    print("Prefix      :", tarball_prefix)
+    def set_gcov(self):
+        """Set the gcov file name."""
+        gcov_path = os.path.join(self.config.download_path, self.name + ".gcov")
+        if os.path.isfile(gcov_path):
+            self.gcov_file = self.name + ".gcov"
 
-    with open(build.download_path + "/Makefile", "w") as file:
-        file.write("PKG_NAME := " + name + "\n")
-        file.write("URL := " + url_argument + "\n")
-        file.write("ARCHIVES :=")
-        for archive in archives:
-            file.write(" {}".format(archive))
-        file.write("\n")
-        file.write("\n")
-        file.write("include ../common/Makefile.common\n")
+    def process_archives(self):
+        """Process extra sources needed by package."""
+        src_objects = []
 
-    shutil.rmtree("{}".format(build.base_path), ignore_errors=True)
-    os.makedirs("{}".format(build.output_path))
-    call("mkdir -p %s" % build.download_path)
-    call(extract_cmd)
+        full_archives = self.archives
+        # Download and extract full list
+        for arch_url, destination in zip(full_archives[::2], full_archives[1::2]):
+            src_path = self.check_or_get_file(arch_url, os.path.basename(arch_url), mode="a")
+            # Create source object and extract archive
+            archive = Source(arch_url, destination, src_path, self.config.default_pattern)
+            # Add archive prefix to list
+            self.config.archive_details[arch_url + "prefix"] = archive.prefix
+            self.prefixes[arch_url] = archive.prefix
+            # Add archive to list
+            src_objects.append(archive)
 
-    path = build.base_path + tarball_prefix
+        return src_objects
 
-    for archive, destination in zip(archives[::2], archives[1::2]):
-        source_tarball_path = check_or_get_file(archive, os.path.basename(archive))
-        if source_tarball_path.lower().endswith('.zip'):
-            tarball_contents = subprocess.check_output(
-                ["unzip", "-l", source_tarball_path], universal_newlines=True)
-            if tarball_contents and len(tarball_contents.splitlines()) > 3:
-                source_tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1]
-            extract_cmd = "unzip -d {0} {1}".format(build.base_path, source_tarball_path)
-        else:
-            extract_cmd, source_tarball_prefix = build_untar(source_tarball_path)
-        buildpattern.archive_details[archive + "prefix"] = source_tarball_prefix
-        call(extract_cmd)
-        tar_files = glob.glob("{0}{1}/*".format(build.base_path, source_tarball_prefix))
-        move_cmd = "mv "
-        for tar_file in tar_files:
-            move_cmd += tar_file + " "
-        move_cmd += '{0}/{1}'.format(path, destination)
-
-        mkdir_cmd = "mkdir -p "
-        mkdir_cmd += '{0}/{1}'.format(path, destination)
-
-        print("mkdir " + mkdir_cmd)
-        call(mkdir_cmd)
-        call(move_cmd)
-
-        sha1 = get_sha1sum(source_tarball_path)
-        with open(build.download_path + "/upstream", "a") as file:
-            file.write(sha1 + "/" + os.path.basename(archive) + "\n")
-
-
-def write_nvr(file):
-    global name
-    global version
-    global release
-    file.write("Name     : " + name + "\n")
-    file.write("Version  : " + version + "\n")
-    file.write("Release  : " + str(release) + "\n")
-    file.write("URL      : " + url + "\n")
-    file.write("Source0  : " + url + "\n")
+    def process(self, filemanager):
+        """Download and process the tarball."""
+        # determine build pattern and build requirements from url
+        self.set_giturl_and_domain()
+        # determine name and version of package
+        self.name_and_version(filemanager)
+        # set gcov file information, must be done after name is set since the gcov
+        # name is created by adding ".gcov" to the package name (if a gcov file
+        # exists)
+        self.set_gcov()
+        # Download and process main source
+        main_src = self.process_main_source(self.url)
+        # Store the detected prefix associated with this file
+        self.prefixes[self.url] = main_src.prefix
+        self.tarball_prefix = main_src.prefix
+        # set global path with tarball_prefix
+        self.path = os.path.join(self.base_path, self.tarball_prefix)
+        # Now that the metadata has been collected print the header
+        self.print_header()
+        # Download and process extra sources: archives
+        archives_src = self.process_archives()
+        # Extract all sources
+        self.extract_sources(main_src, archives_src)

autospec/test.py

@@ -1,167 +0,0 @@
-#!/bin/true
-#
-# test.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Deduce and emmit the patterns for %check
-#
-
-import buildpattern
-import buildreq
-import glob
-import os
-import tarball
-import subprocess
-import config
-
-tests_config = ""
-skip_tests = False
-allow_test_failures = False
-new_pkg = True
-unit_pass_written = False
-
-
-def check_regression(dir):
-
-    global skip_tests
-    if skip_tests:
-        return
-
-    build_log_path = os.path.join(dir, "results/build.log")
-    perl_cmd = ["perl", os.path.dirname(__file__) + "/count.pl", build_log_path]
-    result = subprocess.check_output(perl_cmd)
-    result = result.decode("utf-8")
-    lines = result.strip('\n').split('\n')
-    titles = ['package name', 'total tests', 'tests passing', 'tests failing',
-              'tests skipped', 'expected fail']
-    if len(lines) > 1:
-        for l in lines:
-            split_lines = l.split(',')
-            for x in range(0, len(split_lines)):
-                print(titles[x] + ": " + split_lines[x])
-    else:
-        split_list = lines[0].split(',')
-        for x in range(1, len(split_list)):
-            print(titles[x] + ": " + split_list[x])
-
-
-def scan_for_tests(dir):
-    global tests_config
-    global skip_tests
-    global allow_test_failures
-
-    if skip_tests:
-        return
-
-    if len(tests_config) > 0:
-        return
-
-    makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
-    testsuites = {
-        "makecheck": "make VERBOSE=1 V=1 {}check".format(makeflags),
-        "perlcheck": "make TEST_VERBOSE=1 test",
-        "setup.py": "PYTHONPATH=%{buildroot}/usr/lib/python2.7/site-packages python2 setup.py test",
-        "cmake": "pushd clr-build ; make test ; popd",
-        "rakefile": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrake --trace test TESTOPTS=\"-v\"\npopd",
-        # "rubygems": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nruby -I\"lib:test*\" test*/*_test.rb \nruby -I\"lib:test*\" test*/test_*.rb\npopd",
-        "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd"
-    }
-
-    files = os.listdir(dir)
-
-    if "CMakeLists.txt" in files:
-        makefile_path = os.path.join(dir, "CMakeLists.txt")
-        if not os.path.isfile(makefile_path):
-            return
-        with open(makefile_path, encoding="latin-1") as fp:
-            lines = fp.readlines()
-        for line in lines:
-            if line.find("enable_testing") >= 0:
-                tests_config = testsuites["cmake"]
-                break
-
-    if "Makefile.in" in files:
-        makefile_path = os.path.join(dir, "Makefile.in")
-        if not os.path.isfile(makefile_path):
-            return
-        with open(makefile_path) as fp:
-            lines = fp.readlines()
-        for line in lines:
-            if line.startswith("check:"):
-                tests_config = testsuites["makecheck"]
-                break
-            if line.startswith("test:"):
-                tests_config = testsuites["perlcheck"]
-                break
-    elif "Makefile.am" in files:
-        tests_config = testsuites["makecheck"]
-    elif tarball.name.startswith("rubygem"):
-        if "test" in files or "tests" in files:
-            r_testdir = [f for f in files if "test" in f if "." not in f].pop()
-            pre_test = glob.glob(os.path.join(dir, "test*/test_*.rb"))
-            post_test = glob.glob(os.path.join(dir, "test*/*_test.rb"))
-            tests_config = "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix
-            if len(pre_test) > 0:
-                tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/test_*.rb"
-            if len(post_test) > 0:
-                tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/*_test.rb"
-            tests_config += "\npopd"
-
-        elif "spec" in files:
-            buildreq.add_buildreq("rubygem-rspec")
-            buildreq.add_buildreq("rubygem-rspec-core")
-            buildreq.add_buildreq("rubygem-rspec-expectations")
-            buildreq.add_buildreq("rubygem-rspec-support")
-            buildreq.add_buildreq("rubygem-rspec-mocks")
-            buildreq.add_buildreq("rubygem-devise")
-            buildreq.add_buildreq("rubygem-diff-lcs")
-            tests_config = testsuites["rspec"]
-        elif "Rakefile" in files:
-            with open(os.path.join(dir, "Rakefile"), encoding="ascii", errors="surrogateescape") as fp:
-                setup_contents = fp.read()
-                # if setup_contents.find("task :test") >= 0 or setup_contents.find("task 'test'") >= 0:
-                tests_config = testsuites["rakefile"]
-                buildreq.add_buildreq("ruby")
-                buildreq.add_buildreq("rubygem-rake")
-                buildreq.add_buildreq("rubygem-test-unit")
-                buildreq.add_buildreq("rubygem-minitest")
-    elif "Makefile.PL" in files:
-        tests_config = testsuites["perlcheck"]
-    elif "setup.py" in files:
-        with open(os.path.join(dir, "setup.py"), encoding="ascii", errors="surrogateescape") as fp:
-            setup_contents = fp.read()
-            if "test_suite" in setup_contents or "pbr=True" in setup_contents:
-                tests_config = testsuites["setup.py"]
-    elif buildpattern.default_pattern == "R":
-        tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\nR CMD check --no-manual --no-examples --no-codoc -l %{buildroot}/usr/lib64/R/library " + tarball.rawname
-
-    if "tox.ini" in files:
-        buildreq.add_buildreq("tox")
-        buildreq.add_buildreq("pytest")
-        buildreq.add_buildreq("virtualenv")
-        buildreq.add_buildreq("pluggy")
-        buildreq.add_buildreq("py-python")
-
-    if tests_config != "" and allow_test_failures:
-        if tarball.name.startswith("rubygem"):
-            config_elements = tests_config.split("\n")
-            config_elements.pop()
-            tests_config = "\n".join(config_elements) + " || :\npopd"
-        elif tests_config == testsuites["cmake"]:
-            tests_config = "pushd clr-build ; make test ||: ; popd"
-        else:
-            tests_config = tests_config + " || :"
-        print(tests_config)

autospec/translate.dic

@@ -1,54 +1,82 @@
-babel=Babel
+async_timeout=pypi-async_timeout
+babel=pypi-babel
+backports.ssl-match-hostname=pypi-backports.ssl_match_hostname
 barbicanclient-python=python-barbicanclient
-BeautifulSoup=beautifulsoup4
-cryptography-python=cryptography
+BeautifulSoup=pypi-beautifulsoup4
+cryptography-python=pypi-cryptography
 Crypto-python=pycrypt-python
-dateutil.relativedelta-python=python-dateutil
+dateutil-python=pypi-python_dateutil
+dateutil.relativedelta-python=pypi-python_dateutil
 designateclient-python=python-designateclient
-django-python=Django
-enum-python=enum34-python
+django-python=pypi-django
+enum-python=enum34
 formencode=FormEncode
 Genshi-python=Genshi
+gi.overrides-python=pygobject-python
+gi-python=pygobject-python
+github.com/golang/glog=golang-github-golang-glog
+github.com/gorilla/context=golang-github-gorilla-context
+github.com/mattn/go-sqlite3=golang-github-mattn-go-sqlite3
+github.com/mitchellh/mapstructure=golang-github-mitchellh-mapstructure
+github.com/stretchr/testify/assert=golang-github-stretchr-testify
+glance-store=glance_store
+golang.org/x/net/context=golang-googlecode-go-net
+golang.org/x/net/netutil=golang-googlecode-go-net
+gopkg.in/check.v1=golang-github-go-check-check
+gopkg.in/yaml.v2=golang-github-go-yaml-yaml
 hacking-python=hacking
 httplib2-python=httplib2
 ironicclient-python=python-ironicclient
-jinja2-python=Jinja2
-Jinja2-python=Jinja2
+jinja2=pypi-jinja2
+jinja2-python=pypi-jinja2
+Jinja2-python=pypi-jinja2
+jsonpath_rw_ext=jsonpath-rw-ext
 keystoneclient.auth-python=python-keystoneclient
 keystoneclient.v3-python=python-keystoneclient
 keystonemiddleware-python=keystonemiddleware
 lazy_object_proxy=lazy-object-proxy
 manilaclient-python=python-manilaclient
 mistralclient.api.v2-python=python-mistralclient
-mock=python-mock
 netaddr-python=netaddr
+Opcodes=opcodes
 openstackclient.tests-python=python-openstackclient
 openstack.nose-plugin-python=openstack.nose_plugin
 os_brick.initiator-python=os-brick
 oslo_cache-python=oslo.cache
-oslo.config-python=oslo.config
 oslo_config-python=oslo.config
+oslo.config-python=oslo.config
 oslo_policy-python=oslo.policy
 oslo_reports-python=oslo.reports
 oslo_versionedobjects-python=oslo.versionedobjects
 oslo_versionedobjects.tests-python=oslo.versionedobjects
-pep8-python=pep8
-posix-ipc-python=posix_ipc
-PrettyTable-python=prettytable
-pyaml-python=pyaml
+pep8-python=pypi-pep8
+posix-ipc-python=pypi-posix_ipc
+PrettyTable-python=pypi-prettytable
+pyaml-python=pypi-pyaml
 PyECLib-python=pyeclib
-Pygments-python=Pygments
+pygments=pypi-pygments
+Pygments-python=pypi-pygments
+PyGObject=pygobject
 pylibmc-python=pylibmc
+PyOpenSSL=pypi-pyopenssl
+pyqt5=PyQt5
+pytest_runner=pytest-runner
 python-memcached-python=python-memcached
-setuptools-scm-python=setuptools_scm
+pyyaml=PyYAML
+qtawesome=QtAwesome
+qtpy=QtPy
+restructuredtext-lint=restructuredtext_lint
+semantic-version=semantic_version
+setuptools-scm-python=pypi-setuptools_scm
 simplejson-python=simplejson
-sphinx=Sphinx
-sqlparse-python=sqlparse
+sphinx=pypi-sphinx
+sqlalchemy=pypi-sqlalchemy
+sqlparse-python=pypi-sqlparse
 stevedore-python=stevedore
 testresources-python=testresources
 testscenarios-python=testscenarios
 websocket-client=websocket_client
-WebTest-python=WebTest
+WebTest-python=pypi-webtest
 yaml-python=PyYAML
 zaqarclient.queues.v1-python=python-zaqarclient
 zaqarclient.transport-python=python-zaqarclient

autospec/util.py

@@ -17,15 +17,115 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import hashlib
 import os
+import re
 import shlex
 import subprocess
+import sys
 
 dictionary_filename = os.path.dirname(__file__) + "/translate.dic"
 dictionary = [line.strip() for line in open(dictionary_filename, 'r')]
+os_paths = None
+ERROR_FILE = 'pumpAutospec'
+ERROR_ENV = 'AUTOSPEC_UPDATE'
+
+
+def _log_error(error):
+    write_out(ERROR_FILE, f"{error}\n", mode='a')
+
+
+def _commit_result():
+    if not os.path.isfile(ERROR_FILE):
+        return
+    call(f"git add {ERROR_FILE}", check=False, stderr=subprocess.DEVNULL)
+    call(f"git commit {ERROR_FILE} -m 'Notes update'", check=False, stderr=subprocess.DEVNULL)
+    call("git push", check=False, stderr=subprocess.DEVNULL)
+
+
+def _process_line(line, prev_line, current_patch, reported_patches, error):
+    if m := re.match('^Patch #[0-9]+ .(?P<patch>.*).:', line):
+        current_patch[0] = m.group('patch')
+
+    if m := re.match('Hunk #[0-9]+ FAILED at [0-9]+', line):
+        if current_patch[0] not in reported_patches:
+            _log_error("Patch " + current_patch[0] + " does not apply")
+            reported_patches[current_patch[0]] = True
+            return True
+
+    if m := re.match(".*can't find file to patch at input line ", line):
+        if current_patch[0] not in reported_patches:
+            _log_error("Patch " + current_patch[0] + " does not apply")
+            reported_patches[current_patch[0]] = True
+            return True
+
+    if m := re.match('.*meson.build:[0-9]+:[0-9]+: ERROR: Unknown options: "(?P<option>.*)"', line):
+        _log_error("Unknown meson option: " + m.group('option'))
+        return True
+
+    if m := re.match('Error: package ‘(?P<module>.*)’ .* was found, but', line):
+        _log_error("R package " + m.group('module') + " not found")
+        return True
+
+    if m := re.match('.*CMake Error at .*/CMake', prev_line):
+        if m := re.match('(?P<module>.*) not found', line):
+            _log_error("CMake module " + m.group('module') + "not found")
+            return True
+
+    if m := re.match(r'go: download.*connect: connection refused', line):
+        _log_error("Go online update")
+        return True
+
+    if 'Updating crates.io index' in line:
+        _log_error("Rust crates.io online update")
+        return True
+
+    if "error: '__builtin_ctzs' needs isa option -mbmi" in line:
+        _log_error(" error: '__builtin_ctzs' needs isa option -mbmi")
+        return True
+
+    if "error:" in line and 'Bad exit status from' not in line:
+        m = re.match('.*error:(?P<error>.*)', line)
+        if m and not error:
+            _log_error("Compiler: " + m.group('error'))
+            return True
+
+    if m := re.match(r'Could NOT find (?P<package>.*) .missing', line):
+        _log_error("CMake module " + m.group('package') + " not found")
+        return True
+    if m := re.match(r'Could not find a package configuration file provided by (?P<package>.*) with', line):
+        _log_error("CMake module " + m.group('package') + " not found")
+        return True
+    # Unable to find program 'gperf'
+    if m := re.match(r"Failed to find program ‘(?P<module>.*)’", line):
+        _log_error("Failed to find " + m.group('module'))
+        return True
+    if m := re.match(r"Failed to find ‘(?P<module>.*)’", line):
+        _log_error("Failed to find " + m.group('module'))
+        return True
+
+    return False
+
+
+def _process_build_log(filename):
+    with open_auto(filename, "r") as lfile:
+        lines = lfile.readlines()
+
+    prev_line = ''
+    current_patch = ['']
+    reported_patches = {}
+    error = False
+    for line in lines:
+        if _process_line(line, prev_line, current_patch, reported_patches, error):
+            error = True
+        prev_line = line
+
+    if error:
+        _commit_result()
 
 
 def call(command, logfile=None, check=True, **kwargs):
+    """Subprocess.call convenience wrapper."""
     returncode = 1
     full_args = {
         "args": shlex.split(command),
@@ -55,8 +155,149 @@ def _file_write(self, s):
 
 
 def translate(package):
-    global dictionary
+    """Convert terms to their alternate definition."""
     for item in dictionary:
         if item.startswith(package + "="):
             return item.split("=")[1]
     return package
+
+
+def do_regex(patterns, re_str):
+    """Find a match in multiple patterns."""
+    for p in patterns:
+        match = re.search(p, re_str)
+        if match:
+            return match
+
+
+def get_contents(filename):
+    """Get contents of filename."""
+    with open(filename, "rb") as f:
+        return f.read()
+    return None
+
+
+def get_sha1sum(filename):
+    """Get sha1 sum of filename."""
+    sh = hashlib.sha1()
+    sh.update(get_contents(filename))
+    return sh.hexdigest()
+
+
+def _supports_color():
+    # FIXME: check terminfo instead
+    return sys.stdout.isatty()
+
+
+def _print_message(message, level, color=None):
+    prefix = level
+    if color and _supports_color():
+        # FIXME: use terminfo instead
+        if color == 'red':
+            params = '31;1'
+        elif color == 'green':
+            params = '32;1'
+        elif color == 'yellow':
+            params = '33;1'
+        elif color == 'blue':
+            params = '34;1'
+        prefix = f'\033[{params}m{level}\033[0m'
+    print(f'[{prefix}] {message}')
+
+
+def print_error(message):
+    """Print error, color coded for TTYs."""
+    _print_message(message, 'ERROR', 'red')
+
+
+def print_build_failed():
+    """Print final fatal error, color coded for TTYs."""
+    _print_message('Build failed, aborting', 'FATAL', 'red')
+    try:
+        if os.environ.get(ERROR_ENV):
+            _process_build_log('results/build.log')
+    except Exception:
+        pass
+
+
+def print_fatal(message):
+    """Print fatal error, color coded for TTYs."""
+    _print_message(message, 'FATAL', 'red')
+    if os.environ.get(ERROR_ENV):
+        write_out(ERROR_FILE, f"{message}\n", mode='a')
+        _commit_result()
+
+
+def print_warning(message):
+    """Print warning, color coded for TTYs."""
+    _print_message(message, 'WARNING', 'red')
+
+
+def print_info(message):
+    """Print informational message, color coded for TTYs."""
+    _print_message(message, 'INFO', 'yellow')
+
+
+def print_success(message):
+    """Print success message, color coded for TTYs."""
+    _print_message(message, 'SUCCESS', 'green')
+
+
+def binary_in_path(binary):
+    """Determine if the given binary exists in the provided filesystem paths."""
+    global os_paths
+    if not os_paths:
+        os_paths = os.getenv("PATH", default="/usr/bin:/bin").split(os.pathsep)
+
+    for path in os_paths:
+        if os.path.exists(os.path.join(path, binary)):
+            return True
+    return False
+
+
+def write_out(filename, content, mode="w"):
+    """File.write convenience wrapper."""
+    with open_auto(filename, mode) as require_f:
+        require_f.write(content)
+
+
+def open_auto(*args, **kwargs):
+    """Open a file with UTF-8 encoding.
+
+    Open file with UTF-8 encoding and "surrogate" escape characters that are
+    not valid UTF-8 to avoid data corruption.
+    """
+    # 'encoding' and 'errors' are fourth and fifth positional arguments, so
+    # restrict the args tuple to (file, mode, buffering) at most
+    assert len(args) <= 3
+    assert 'encoding' not in kwargs
+    assert 'errors' not in kwargs
+    return open(*args, encoding="utf-8", errors="surrogateescape", **kwargs)
+
+
+def globlike_match(filename, match_name):
+    """Compare the filename to the match_name in a way that simulates the shell glob '*'."""
+    fsplit = filename.split('/')
+    if len(fsplit) != len(match_name):
+        return False
+    match = True
+    for fpart, mpart in zip(fsplit, match_name):
+        if fpart != mpart:
+            if '*' not in mpart:
+                match = False
+                break
+            if len(mpart) > len(fpart) + 1:
+                match = False
+                break
+            mpl, mpr = mpart.split('*')
+            try:
+                if fpart.index(mpl) != 0:
+                    match = False
+                    break
+                if fpart.rindex(mpr) != len(fpart) - len(mpr):
+                    match = False
+                    break
+            except ValueError:
+                match = False
+                break
+    return match

requirements.txt

@@ -0,0 +1,6 @@
+flake8>=3.4.0
+pycurl>=7.43.0
+toml>=0.9.0
+mock>=2.0.0
+coverage>=4.4.1
+requests>=2.18.4

setup.cfg

@@ -1,2 +1,12 @@
 [egg_info]
 tag_build = 
+[pycodestyle]
+ignore = E501
+
+[coverage:run]
+# omit tests and site-packages content
+omit = tests/*,*site-packages*,*site.py
+
+[flake8]
+max-line-length = 199
+ignore = B902, E722, W503

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.0.0"
+version = "1.2.0"
 
 def readme():
     with open("README.rst") as f:
@@ -21,10 +21,13 @@ setup(name="autospec",
             'Topic :: Software Development :: Build Tools',
             'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
             # Python versions supported.
+            'Programming Language :: Python :: 3 :: Only',
             'Programming Language :: Python :: 3',
             'Programming Language :: Python :: 3.2',
             'Programming Language :: Python :: 3.3',
             'Programming Language :: Python :: 3.4',
+            'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
         ],
       include_package_data = True,
 )

tests/COPYING_TEST

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

tests/README.rst

@@ -0,0 +1,23 @@
+================
+Autospec Testing
+================
+
+Code Style
+==========
+
+Autospec changes are scanned to vet code style issues with the ``flake8`` tool.
+To check for issues, run ``make check`` from the root of the autospec source
+tree, which executes ``flake8`` with appropriate arguments.
+
+Unit
+====
+
+Autospec ships with several test modules that correspond to individual modules
+from the toplevel ``autospec`` directory.
+
+Each module can be tested in isolation by running ``make test_<MODULE>``, where
+``<MODULE>`` corresponds to the module name. For example, ``make
+test_pkg_integrity`` runs unit tests for the ``pkg_integrity.py`` module.
+
+To run *all* unit tests, run ``make unittests``. If all tests pass, a code
+coverage report is also generated.

tests/builderrors

@@ -0,0 +1,103 @@
+    !  swig is not installed|perl(swig)
+    ExtUtils::Depends not installed|perl(ExtUtils::Depends)
+  Could not find swig|swig
+  Did not find swig|swig
+ exec: swig: not found|swig
+ mate-common.m4 not found|mate-common-dev
+-- Could NOT find swig|swig
+-- Could not find wpa_supplicant|wpa_supplicant
+-- Could not find xdg-open|xdg-utils
+-- swig not found.|swig
+etc etc /usr/bin/swig   not found|swig
+/bin/ld: cannot find -lz|zlib-dev
+/usr/bin/env: swig: No such file or directory|swig
+/usr/bin/python: No module named swig|swig
+Add the installation prefix of "swig" to CMAKE_PREFIX_PATH|swig
+By not providing "Findswig.cmake" in CMAKE_MODULE_PATH this|swig
+By not providing "swig.cmake" in CMAKE_MODULE_PATH this project|swig
+C library 'swig' not found|swig
+CMake Error at cmake/modules/swig.cmake|swig
+Can't locate swig in @INC (you may need to install the swig module)|perl(swig)
+Cannot find swig|swig
+Checking for swig     : not found|swig
+Checking for swig  >=    1.2   : not found|swig
+Checking for swig >= 1.2   : not found|swig
+Checking for swig development files... No|swig
+Checking for swig...no|swig
+Checking for swig: not found|swig
+Could NOT find swig|swig
+Could not find wpa_supplicant|wpa_supplicant
+Could not find xdg-open|xdg-utils
+Could not find a package configuration file provided by "swig" etc etc|swig
+Could not find suitable distribution for Requirement.parse('swig')|swig
+Dependency swig found: NO|swig
+Dependency testpkg found: NO (tried pkgconfig and cmake)|pkgconfig(testpkg)
+Dependency testpkg found: NO (tried pkgconfig)|pkgconfig(testpkg)
+Download error on https://pypi.python.org/simple/swig/|pypi(swig)
+Downloading https://test.python.org/packages/pkg/t/swig/1.23|swig
+ERROR: dependencies 'swig' are not available for package 'something'|R-swig
+ERROR: dependencies 'swig', 'swig2' are not available for package 'something'|R-swig
+ERROR: dependencies ‘swig’ are not available for package ‘something’|R-swig
+ERROR: dependencies ‘swig’, ‘swig2’ are not available for package ‘something’|R-swig
+ERROR: dependency 'swig' is not available for package 'something'|R-swig
+ERROR: dependency ‘swig’ is not available for package ‘something’|R-swig
+Error: Unable to find swig|swig
+Error: package 'swig' required by|R-swig
+Error: package ‘swig’ required by|R-swig
+ImportError: No module named 'swig'|pypi(swig)
+ImportError: No module named swig|swig
+ImportError: No module named swig|pypi(swig)
+ImportError: cannot import name swig|swig
+ImportError: swig module missing|swig
+ImportError:..*: No module named swig|pypi(swig)
+ModuleNotFoundError: No module named swig|pypi(swig)
+ModuleNotFoundError: No module named 'foo-bar'|pypi(foo_bar)
+Native dependency 'swig' not found|pkgconfig(swig)
+No local packages or working download links found for swig|pypi(swig)
+No matching distribution found for swig|pypi(swig)
+No package 'swig' found|pkgconfig(swig)
+No rule to make target `swig', etc|swig
+Package 'swig', required by 'something', not found|pkgconfig(swig)
+Package which this enhances but not available for checking: 'swig'|R-swig
+Package which this enhances but not available for checking: ‘swig’|R-swig
+Perhaps you should add the directory containing `swig.pc'|pkgconfig(swig)
+Program swig found: NO|swig
+Target 't' can't be generated as 'swig' could not be found|swig
+Unable to `import swig`|swig
+Unable to find 'swig'|swig
+Warning: no usable swig found|swig
+Warning: prerequisite swig 8 not found.|perl(swig)
+You need swig to build this program.|swig
+"swig" with any of the following names|swig
+checking for library containing swig... no|swig
+checking for perl module swig 7... no|perl(swig)
+checking for something in swig... no|swig
+checking for swig in default path... not found|swig
+checking for swig support... no|swig
+checking for swig support... no|swig
+checking for swig with pkg-config... no|swig
+checking for swig... configure: error|swig
+checking for swig... no|swig
+checking for swig... not found|swig
+checking for swig... not_found|swig
+checking swig... no|swig
+configure: error: Cannot find swig. Make sure|swig
+configure: error: Unable to locate swig|swig
+configure: error: pkg-config missing swig|swig
+configure: error: swig is required to build|swig
+configure: error: swig not found|swig
+dependency testpkg found: NO (tried pkgconfig and cmake)|pkgconfig(testpkg)
+dependency testpkg found: NO (tried pkgconfig)|pkgconfig(testpkg)
+fatal error: swig: No such file or directory|swig
+make: swig: Command not found|swig
+make: help2man: No such file or directory|help2man
+swig 8 is required to configure this module; please install it or upgrade your CPAN/CPANPLUS shell.|swig
+swig tool not found or not executable|swig
+swig validation tool not found or not executable|swig
+swig: command not found|swig
+there is no package called 'swig'|R-swig
+there is no package called ‘swig’|R-swig
+unable to execute 'swig': No such file or directory|swig
+warning: failed to load external entity "/usr/share/sgml/docbook/xsl-stylesheets/something"|docbook-xml
+which: no swig in (/usr/bin/swig/)|swig
+you may need to install the swig module|perl(swig)

tests/test_abireport.py

@@ -0,0 +1,273 @@
+import unittest
+import abireport
+
+
+def mock_return(retval):
+    """
+    Simple mock method to set return value of a function
+    """
+    def mock_fn(_):
+        return retval
+
+    return mock_fn
+
+
+class TestAbireport(unittest.TestCase):
+
+    def test_get_soname(self):
+        """
+        Test get_soname function with valid get_output results
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return('SONAME               libtest.so.14')
+        self.assertEqual(abireport.get_soname('test'), 'libtest.so.14')
+        abireport.get_output = backup
+
+    def test_get_soname_none(self):
+        """
+        Test get_soname function with no get_output results
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return('')
+        self.assertEqual(abireport.get_soname('test'), None)
+        abireport.get_output = backup
+
+    def test_get_soname_error(self):
+        """
+        Test get_soname function with exception raised in get_output
+        """
+        backup = abireport.get_output
+
+        def mock_get_output(cmd):
+            raise Exception('testing exception')
+
+        abireport.get_output = mock_get_output
+        self.assertEqual(abireport.get_soname('test'), None)
+        abireport.get_output = backup
+
+    def test_get_shared_dependencies(self):
+        """
+        Test get_shared_dependencies function with valid get_output results
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return(READELF1)
+        results = set(['libdl.so.2', 'libz.so.1', 'libexpat.so.1',
+                       'libstdc++.so.6', 'libm.so.6', 'libgcc_s.so.1',
+                       'libc.so.6'])
+        self.assertEqual(abireport.get_shared_dependencies('test'), results)
+        abireport.get_output = backup
+
+    def test_get_shared_dependencies_none(self):
+        """
+        Test get_shared_dependencies with no matches
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return(READELF2)
+        self.assertEqual(abireport.get_shared_dependencies('test'), set())
+        abireport.get_output = backup
+
+    def test_is_dynamic_binary(self):
+        """
+        Test is_dynamic_binary function for True return
+        """
+        backup_isfile = abireport.os.path.isfile
+        backup_exists = abireport.os.path.exists
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.isfile = mock_return(True)
+        abireport.os.path.exists = mock_return(True)
+        abireport.get_file_magic = mock_return(FILEMAGIC1)
+
+        self.assertTrue(abireport.is_dynamic_binary('test'))
+
+        abireport.os.path.isfile = backup_isfile
+        abireport.os.path.exists = backup_exists
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_is_dynamic_binary_false(self):
+        """
+        Test is_dynamic_binary function for False return
+        """
+        backup_exists = abireport.os.path.exists
+        backup_isfile = abireport.os.path.isfile
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.exists = mock_return(True)
+        abireport.os.path.isfile = mock_return(True)
+        abireport.get_file_magic = mock_return('This is not a matched string')
+
+        self.assertFalse(abireport.is_dynamic_binary('test'))
+
+        abireport.os.path.exists = backup_exists
+        abireport.os.path.isfile = backup_isfile
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_is_file_valid(self):
+        """
+        Test is_file_valid function with valid file magic
+        """
+        backup_exists = abireport.os.path.exists
+        backup_islink = abireport.os.path.islink
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.exists = mock_return(True)
+        abireport.os.path.islink = mock_return(False)
+        abireport.get_file_magic = mock_return(FILEMAGIC1)
+
+        self.assertTrue(abireport.is_file_valid('test'))
+
+        abireport.os.path.exists = backup_exists
+        abireport.os.path.islink= backup_islink
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_is_file_valid_false(self):
+        """
+        Test is_file_valid function with valid file magic
+        """
+        backup_exists = abireport.os.path.exists
+        backup_islink = abireport.os.path.islink
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.exists = mock_return(True)
+        abireport.os.path.islink = mock_return(False)
+        abireport.get_file_magic = mock_return('This is not a matched string')
+
+        self.assertFalse(abireport.is_file_valid('test'))
+
+        abireport.os.path.exists = backup_exists
+        abireport.os.path.islink= backup_islink
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_dump_symbols(self):
+        """
+        Test dump_symbols function with valid nm output
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return(NM1)
+        results = set(['WXMPIterator_DecrementRefCount_1',
+                       'WXMPIterator_IncrementRefCount_1',
+                       '_Z13XMP_InitMutexP15pthread_mutex_t',
+                       '_Z13XMP_TermMutexR15pthread_mutex_t',
+                       '_Z14CloneOffspringPK8XMP_NodePS_',
+                       '_Z14SortNamedNodesRSt6vectorIP8XMP_NodeSaIS1_EE',
+                       '_Z15CompareSubtreesRK8XMP_NodeS1_',
+                       '_ZN5Exiv211focalLengthERKNS_8ExifDataE',
+                       '_ZN5Exiv211getRationalEPKhNS_9ByteOrderE',
+                       '_ZN5Exiv211isBigEndianEv',
+                       '_ZN5Exiv211orientationERKNS_8ExifDataE',
+                       '_ZN5Exiv28GifImageC2ESt8auto_ptrINS_7BasicIoEE'])
+        self.assertEqual(abireport.dump_symbols('test'), results)
+        abireport.get_output = backup
+
+    def test_dump_symbols_exit(self):
+        """
+        Test dump_symbols function with fatal exception
+        """
+        backup = abireport.get_output
+
+        def mock_get_output(_):
+            raise Exception('This is a test exception')
+
+        abireport.get_output = mock_get_output
+        with self.assertRaises(SystemExit) as dumpsymbols:
+            abireport.dump_symbols('test')
+
+        self.assertEqual(dumpsymbols.exception.code, 1)
+
+
+READELF1 = """
+
+Dynamic section at offset 0x2d0788 contains 31 entries:
+  Tag        Type                         Name/Value
+ 0x0000000000000001 (NEEDED)             Shared library: [libdl.so.2]
+ 0x0000000000000001 (NEEDED)             Shared library: [libz.so.1]
+ 0x0000000000000001 (NEEDED)             Shared library: [libexpat.so.1]
+ 0x0000000000000001 (NEEDED)             Shared library: [libstdc++.so.6]
+ 0x0000000000000001 (NEEDED)             Shared library: [libm.so.6]
+ 0x0000000000000001 (NEEDED)             Shared library: [libgcc_s.so.1]
+ 0x0000000000000001 (NEEDED)             Shared library: [libc.so.6]
+ 0x000000000000000e (SONAME)             Library soname: [libexiv2.so.14]
+ 0x000000000000000f (RPATH)              Library rpath: [/usr/lib]
+ 0x000000000000000c (INIT)               0xca320
+ 0x000000000000000d (FINI)               0x20e6cc
+ 0x0000000000000019 (INIT_ARRAY)         0x2a5a70
+ 0x000000000000001b (INIT_ARRAYSZ)       400 (bytes)
+ 0x000000000000001a (FINI_ARRAY)         0x2a5c00
+ 0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
+ 0x0000000000000004 (HASH)               0x1f0
+ 0x0000000000000005 (STRTAB)             0x23f10
+ 0x0000000000000006 (SYMTAB)             0x8ac0
+ 0x000000000000000a (STRSZ)              252569 (bytes)
+ 0x000000000000000b (SYMENT)             24 (bytes)
+ 0x0000000000000003 (PLTGOT)             0x2d19b8
+ 0x0000000000000007 (RELA)               0x63f48
+ 0x0000000000000008 (RELASZ)             418776 (bytes)
+ 0x0000000000000009 (RELAENT)            24 (bytes)
+ 0x0000000000000018 (BIND_NOW)           
+ 0x000000006ffffffb (FLAGS_1)            Flags: NOW
+ 0x000000006ffffffe (VERNEED)            0x63e08
+ 0x000000006fffffff (VERNEEDNUM)         5
+ 0x000000006ffffff0 (VERSYM)             0x619aa
+ 0x000000006ffffff9 (RELACOUNT)          11626
+ 0x0000000000000000 (NULL)               0x0
+"""
+
+READELF2 = """
+
+Dynamic section at offset 0x2d0788 contains 31 entries:
+  Tag        Type                         Name/Value
+ 0x000000000000000e (SONAME)             Library soname: [libexiv2.so.14]
+ 0x000000000000000f (RPATH)              Library rpath: [/usr/lib]
+ 0x000000000000000c (INIT)               0xca320
+ 0x000000000000000d (FINI)               0x20e6cc
+ 0x0000000000000019 (INIT_ARRAY)         0x2a5a70
+ 0x000000000000001b (INIT_ARRAYSZ)       400 (bytes)
+ 0x000000000000001a (FINI_ARRAY)         0x2a5c00
+ 0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
+ 0x0000000000000004 (HASH)               0x1f0
+ 0x0000000000000005 (STRTAB)             0x23f10
+ 0x0000000000000006 (SYMTAB)             0x8ac0
+ 0x000000000000000a (STRSZ)              252569 (bytes)
+ 0x000000000000000b (SYMENT)             24 (bytes)
+ 0x0000000000000003 (PLTGOT)             0x2d19b8
+ 0x0000000000000007 (RELA)               0x63f48
+ 0x0000000000000008 (RELASZ)             418776 (bytes)
+ 0x0000000000000009 (RELAENT)            24 (bytes)
+ 0x0000000000000018 (BIND_NOW)           
+ 0x000000006ffffffb (FLAGS_1)            Flags: NOW
+ 0x000000006ffffffe (VERNEED)            0x63e08
+ 0x000000006fffffff (VERNEEDNUM)         5
+ 0x000000006ffffff0 (VERSYM)             0x619aa
+ 0x000000006ffffff9 (RELACOUNT)          11626
+ 0x0000000000000000 (NULL)               0x0
+"""
+
+FILEMAGIC1 = '/path/to/so/libtest.so.1: ELF 64-bit LSB shared object, ' \
+             'x86-64, version 1 (SYSV), dynamically linked, '           \
+             'BuildID[sha1]=c99853fe0c5d8f3c7550848edf25fe7782fb22c3, stripped'
+
+NM1 = """
+00000000002f1d88 B voidStringPtr
+00000000002f1d90 B voidVoidPtr
+00000000002f1ce0 B void_wResult
+00000000001d1a40 T WXMPIterator_DecrementRefCount_1
+00000000001d19f0 T WXMPIterator_IncrementRefCount_1
+00000000001daa80 T _Z13XMP_InitMutexP15pthread_mutex_t
+00000000001daaa0 T _Z13XMP_TermMutexR15pthread_mutex_t
+00000000001db900 T _Z14CloneOffspringPK8XMP_NodePS_
+00000000001df4e0 A _Z14SortNamedNodesRSt6vectorIP8XMP_NodeSaIS1_EE
+00000000001dc450 T _Z15CompareSubtreesRK8XMP_NodeS1_
+000000000011d260 A _ZN5Exiv211focalLengthERKNS_8ExifDataE
+00000000001b0510 T _ZN5Exiv211getRationalEPKhNS_9ByteOrderE
+0000000000197160 T _ZN5Exiv211isBigEndianEv
+000000000011c8e0 T _ZN5Exiv211orientationERKNS_8ExifDataE
+00000000002c7000 D _ZN5Exiv211sectionInfoE
+00000000002c45e0 D _ZN5Exiv212xmpXmpBJInfoE
+0000000000137660 T _ZN5Exiv28GifImageC2ESt8auto_ptrINS_7BasicIoEE
+00000000002658d8 R _ZN5Exiv28Internal10canonCfCfgE
+0000000000265960 R _ZN5Exiv28Internal10canonCsCfgE
+00000000002ebac0 B _ZN5Exiv28Internal14SigmaMakerNote8tagInfo_E
+"""
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_build.py

@@ -0,0 +1,508 @@
+import unittest
+import tempfile
+import os
+from unittest.mock import patch, mock_open, MagicMock
+import build
+import buildreq
+import config
+import files
+import tarball
+
+class TestBuildpattern(unittest.TestCase):
+
+    def test_simple_pattern_pkgconfig(self):
+        """
+        Test simple_pattern_pkgconfig with match
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern_pkgconfig('line to test for testpkg.xyz',
+                                     r'testpkg.xyz',
+                                     'testpkg',
+                                     False,
+                                     reqs)
+        self.assertIn('pkgconfig(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_simple_pattern_pkgconfig_32bit(self):
+        """
+        Test simple_pattern_pkgconfig with match and 32bit option set
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern_pkgconfig('line to test for testpkg.zyx',
+                                     r'testpkg.zyx',
+                                     'testpkgz',
+                                     True,
+                                     reqs)
+        self.assertIn('pkgconfig(32testpkgz)', reqs.buildreqs)
+        self.assertIn('pkgconfig(testpkgz)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_simple_pattern_pkgconfig_no_match(self):
+        """
+        Test simple_pattern_pkgconfig with no match, nothing should be modified
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern_pkgconfig('line to test for somepkg.xyz',
+                                     r'testpkg.xyz',
+                                     'testpkg',
+                                     False,
+                                     reqs)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_simple_pattern(self):
+        """
+        Test simple_pattern with match. The main difference between
+        simple_pattern and simple_pattern_pkgconfig is the string that is added
+        to buildreq.buildreqs.
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern('line to test for testpkg.xyz',
+                           r'testpkg.xyz',
+                           'testpkg',
+                           reqs)
+        self.assertIn('testpkg', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_simple_pattern_no_match(self):
+        """
+        Test simple_pattern with no match, nothing should be modified
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern('line to test for somepkg.xyz',
+                           r'testpkg.xyz',
+                           'testpkg',
+                           reqs)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_failed_pattern_no_match(self):
+        """
+        Test failed_pattern with no match
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: somepkg', conf, reqs, r'(test)', 0)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_failed_pattern_no_buildtool(self):
+        """
+        Test failed_pattern with buildtool unset and initial match, but no
+        match in failed_commands.
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg', conf, reqs, r'(test)', 0)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_failed_pattern_no_buildtool_match(self):
+        """
+        Test failed_pattern with buildtool unset and match in failed_commands
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        conf.setup_patterns()
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: lex', conf, reqs, r'(lex)', 0)
+        self.assertIn('flex', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_pkgconfig(self):
+        """
+        Test failed_pattern with buildtool set to pkgconfig
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.xyz',
+                             conf,
+                             reqs,
+                             r'(testpkg)',
+                             0,  # verbose=0
+                             buildtool='pkgconfig')
+        self.assertIn('pkgconfig(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_R(self):
+        """
+        Test failed_pattern with buildtool set to R
+        """
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.r',
+                             conf,
+                             reqs,
+                             r'(testpkg)',
+                             0,  # verbose=0
+                             buildtool='R')
+        self.assertIn('R-testpkg', reqs.buildreqs)
+        self.assertNotIn('R-testpkg', reqs.requires[None])
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_perl(self):
+        """
+        Test failed_pattern with buildtool set to perl
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.pl',
+                             conf,
+                             reqs,
+                             r'(testpkg)',
+                             0,  # verbose=0
+                             buildtool='perl')
+        self.assertIn('perl(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_pypi(self):
+        """
+        Test failed_pattern with buildtool set to pypi
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.py',
+                           conf,
+                           reqs,
+                           r'(testpkg)',
+                           0,  # verbose=0
+                           buildtool='pypi')
+        self.assertIn('pypi(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_buildroot_log_fail(self):
+        """
+        Test parse_buildroot_log with a test log indicating failure due to
+        missing dependencies ('foobar' and 'foobarbaz')
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        def mock_wrap_fatal():
+            output = ['']
+            def mock_print_fatal(msg):
+                output[0] = msg
+            return output, mock_print_fatal
+
+        result, mock_print_fatal = mock_wrap_fatal()
+
+        exit_backup = build.sys.exit
+        call_backup = build.util.call
+        print_backup = build.util.print_fatal
+        build.sys.exit = MagicMock()
+        build.util.call = mock_util_call
+        build.util.print_fatal = mock_print_fatal
+
+        open_name = 'build.util.open_auto'
+        content = "line1\nDEBUG util.py:399:  No matching package to install: 'foobar'\nDEBUG util.py:399:  No matching package to install: 'foobarbaz'\nline 4"
+        m_open = mock_open(read_data=content)
+        pkg = build.Build()
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_buildroot_log('testname', 1)
+
+        build.util.print_fatal = print_backup
+        build.util.call = call_backup
+        mock_exit = build.sys.exit
+        build.sys.exit = exit_backup
+
+        self.assertEqual(result[0], 'Cannot resolve dependency name: foobar\nCannot resolve dependency name: foobarbaz')
+        mock_exit.assert_called_once()
+
+    def test_parse_buildroot_log_pass(self):
+        """
+        Test parse_buildroot_log with a test log indicating no failures
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+
+        open_name = 'build.util.open_auto'
+        content = "line 1\nline 2\nline 3\nline 4"
+        m_open = mock_open(read_data=content)
+        pkg = build.Build()
+
+        result = True
+        with patch(open_name, m_open, create=True):
+            pkg.parse_buildroot_log('testname', 1)
+
+        build.util.call = call_backup
+
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_parse_buildroot_log_noop(self):
+        """
+        Test parse_buildroot_log when parsing should be skipped (i.e. mock
+        returned 0)
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+
+        open_name = 'build.util.open_auto'
+        content = "line 1\nline 2\nline 3\nline 4"
+        m_open = mock_open(read_data=content)
+        pkg = build.Build()
+
+        result = True
+        with patch(open_name, m_open, create=True):
+            result = pkg.parse_buildroot_log('testname', 0)
+
+        build.util.call = call_backup
+
+        self.assertTrue(result)
+
+    def test_parse_build_results_patch(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        a backport patch no longer applying
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        def mock_conf_remove_backport_patch(patch):
+            del patch
+            return 1
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        conf.remove_backport_patch = mock_conf_remove_backport_patch
+        conf.patches = ['backport-test.patch']
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\nPatch #1 (backport-test.patch):\nSkipping patch.'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_build_results_pkgconfig(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        missing qmake package (pkgconfig error)
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        conf.config_opts['32bit'] = True
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\nwhich: no qmake\nexiting'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertIn('pkgconfig(Qt)', reqs.buildreqs)
+        self.assertIn('pkgconfig(32Qt)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_build_results_simple_pats(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        missing httpd-dev package (simple pat error)
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\nchecking for Apache test module support\nexiting'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertIn('httpd-dev', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_build_results_failed_pats(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        missing package.
+        """
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        open_auto_backup = build.util.open_auto
+        build.util.call = MagicMock(return_value=None)
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        with open('tests/builderrors', 'r') as f:
+            builderrors = f.readlines()
+            for error in builderrors:
+                if not error.startswith('#'):
+                    input, output = error.strip('\n').split('|')
+                    reqs.buildreqs = set()
+                    build.util.open_auto = mock_open(read_data=input)
+                    pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+                    self.assertIn(output, reqs.buildreqs)
+                    self.assertGreater(pkg.must_restart, 0)
+
+        # Restoring functions
+        build.util.call = call_backup
+        build.util.open_auto = open_auto_backup
+
+    def test_parse_build_results_files(self):
+        """
+        Test parse_build_results with a test log indicating files are missing
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\n' \
+                  'Installed (but unpackaged) file(s) found:\n' \
+                  '/usr/testdir/file\n' \
+                  '/usr/testdir/file1\n' \
+                  '/usr/testdir/file2\n' \
+                  'RPM build errors\n' \
+                  'errors here\n'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertEqual(fm.files,
+                         set(['/usr/testdir/file',
+                              '/usr/testdir/file1',
+                              '/usr/testdir/file2']))
+        # one for each file added
+        self.assertEqual(pkg.must_restart, 0)
+        self.assertEqual(pkg.file_restart, 3)
+
+    def test_parse_build_results_banned_files(self):
+        """
+        Test parse_build_results with a test log indicating banned files are missing
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\n' \
+                  'Installed (but unpackaged) file(s) found:\n' \
+                  '/opt/file\n' \
+                  '/usr/etc/file\n' \
+                  '/usr/local/file\n' \
+                  '/usr/src/file\n' \
+                  '/var/file\n' \
+                  'RPM build errors\n' \
+                  'errors here\n'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertEqual(fm.has_banned, True)
+        # check no files were added
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_get_mock_cmd_without_consolehelper(self):
+        """
+        Test get_mock_cmd when /usr/bin/mock doesn't point to consolehelper
+        """
+        def mock_realpath(path):
+            return path
+
+        realpath_backup = build.os.path.realpath
+
+        build.os.path.realpath = mock_realpath
+
+        mock_cmd = build.get_mock_cmd()
+
+        build.os.path.realpath = realpath_backup
+
+        self.assertEqual(mock_cmd, 'sudo /usr/bin/mock')
+
+    def test_get_mock_cmd_with_consolehelper(self):
+        """
+        Test get_mock_cmd when /usr/bin/mock points to consolehelper
+        """
+        def mock_realpath(path):
+            return '/usr/bin/consolehelper'
+
+        realpath_backup = build.os.path.realpath
+
+        build.os.path.realpath = mock_realpath
+
+        mock_cmd = build.get_mock_cmd()
+
+        build.os.path.realpath = realpath_backup
+
+        self.assertEqual(mock_cmd, '/usr/bin/mock')
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_buildreq.py

@@ -0,0 +1,950 @@
+import unittest
+import tempfile
+import os
+from unittest.mock import MagicMock, mock_open, patch
+import json
+import io
+import buildreq
+import config
+import pypidata
+
+bak_get_pypi_name = pypidata.get_pypi_name
+def get_pypi_name_wrapper(name, miss=None):
+    """Ignore missing packags in pypi"""
+    return bak_get_pypi_name(name)
+
+
+class TestBuildreq(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Test setup method to reset the buildreq module
+        """
+        self.reqs = buildreq.Requirements("")
+        self.reqs.banned_buildreqs.add('bannedreq')
+
+    def test_add_buildreq(self):
+        """
+        Test add_buildreq with unbanned new req. Follow up by asserting that
+        trying to add the same req a second time results in a False return.
+        """
+        self.assertTrue(self.reqs.add_buildreq('testreq'))
+        self.assertIn('testreq', self.reqs.buildreqs)
+        self.assertFalse(self.reqs.add_buildreq('testreq'))
+
+    def test_add_buildreq_banned(self):
+        """
+        Test add_buildreq with banned new req
+        """
+        self.assertFalse(self.reqs.add_buildreq('bannedreq'))
+        self.assertNotIn('bannedreq', self.reqs.buildreqs)
+
+    def test_ban_requires(self):
+        """
+        Test ban_requires with req already present in requires
+        """
+        self.reqs.requires[None] = set(['testreq'])
+        self.reqs.ban_requires('testreq')
+        self.assertNotIn('testreq', self.reqs.requires[None])
+
+    def test_ban_requires_subpkg(self):
+        """
+        Test ban_requires on a subpkg with req already present in requires
+        """
+        self.reqs.requires['subpkg'] = set(['testreq'])
+        self.reqs.ban_requires('testreq', subpkg='subpkg')
+        self.assertNotIn('testreq', self.reqs.requires['subpkg'])
+
+    def test_add_requires(self):
+        """
+        Test add_requires with unbanned new req already present in
+        buildreqs but not yet present in requires
+        """
+        self.reqs.add_buildreq('testreq')
+        self.assertTrue(self.reqs.add_requires('testreq', ['testreq']))
+        self.assertIn('testreq', self.reqs.requires[None])
+
+    def test_add_requires_subpkg(self):
+        """
+        Test add_requires on a subpkg with unbanned new req already present in
+        buildreqs but not yet present in requires
+        """
+        self.reqs.add_buildreq('testreq')
+        self.assertTrue(self.reqs.add_requires('testreq', ['testreq'], subpkg='subpkg'))
+        self.assertIn('testreq', self.reqs.requires['subpkg'])
+
+    def test_add_requires_not_in_buildreqs(self):
+        """
+        Test add_requires with unbanned new req not present in buildreqs.
+        """
+        self.assertFalse(self.reqs.add_requires('testreq', []))
+        self.assertNotIn('testreq', self.reqs.requires[None])
+
+    def test_add_banned_requires(self):
+        """
+        Test add_requires with banned new req (override buildreq).
+        """
+        self.assertFalse(self.reqs.add_requires('pypi(nose)', [], override=True))
+        self.assertNotIn('testreq', self.reqs.requires[None])
+
+    def test_ban_provides(self):
+        """
+        Test ban_provides with prov already present in provides
+        """
+        self.reqs.provides[None] = set(['testreq'])
+        self.reqs.ban_provides('testreq')
+        self.assertNotIn('testreq', self.reqs.provides[None])
+
+    def test_ban_provides_subpkg(self):
+        """
+        Test ban_provides on a subpkg with prov already present in provides
+        """
+        self.reqs.provides['subpkg'] = set(['testreq'])
+        self.reqs.ban_provides('testreq', subpkg='subpkg')
+        self.assertNotIn('testreq', self.reqs.provides['subpkg'])
+
+    def test_add_provides(self):
+        """
+        Test add_provides with unbanned new prov
+        """
+        self.assertTrue(self.reqs.add_provides('testreq'))
+        self.assertIn('testreq', self.reqs.provides[None])
+
+    def test_add_provides_subpkg(self):
+        """
+        Test add_provides on a subpkg with unbanned new prov
+        """
+        self.assertTrue(self.reqs.add_provides('testreq', subpkg='subpkg'))
+        self.assertIn('testreq', self.reqs.provides['subpkg'])
+
+    def test_add_pkgconfig_buildreq(self):
+        """
+        Test add_pkgconfig_buildreq with config_opts['32bit'] set to False
+        """
+        self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', False))
+        self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs)
+
+    def test_add_pkgconfig_buildreq_32bit(self):
+        """
+        Test add_pkgconfig_buildreq with config_opts['32bit'] set to True
+        """
+        self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', True))
+        self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs)
+        self.assertIn('pkgconfig(32testreq)', self.reqs.buildreqs)
+
+    def test_configure_ac_line(self):
+        """
+        Test configure_ac_line with standard pattern
+        """
+        self.reqs.configure_ac_line('AC_CHECK_FUNC([tgetent])', False)
+        self.assertIn('ncurses-devel', self.reqs.buildreqs)
+
+    def test_configure_ac_line_comment(self):
+        """
+        Test configure_ac_line with commented line
+        """
+        self.reqs.configure_ac_line('# AC_CHECK_FUNC([tgetent])', False)
+        self.assertEqual(self.reqs.buildreqs, set())
+
+    def test_configure_ac_line_pkg_check_modules(self):
+        """
+        Test the somewhat complicated logic of configure_ac_line check for the
+        PKG_CHECK_MODULES((.*?)) line.
+        """
+        self.reqs.configure_ac_line(
+            'PKG_CHECK_MODULES(prefix, '
+            '[module > 2 module2 < 2], '
+            'action-if-found, action-if-not-found)', False)
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(module)', 'pkgconfig(module2)']))
+
+    def test_configure_ac_line_xdt_check_package(self):
+        """
+        Test configure_ac_line for the XFCE version of PKG_CHECK_MODULES
+        """
+        self.reqs.configure_ac_line(
+            'XDT_CHECK_PACKAGE(prefix, '
+            '[module = 2 module2 > 9], '
+            'action-if-found, action-if-not-found)', False)
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(module)', 'pkgconfig(module2)']))
+
+    def test_configure_ac_line_pkg_check_exists(self):
+        """
+        Test configure_ac_line for the PKG_CHECK_EXISTS macro
+        """
+        self.reqs.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], '
+                                   'action-if-found, '
+                                   'action-if-not-found)', False)
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(module1)', 'pkgconfig(module2)']))
+
+    def test_parse_configure_ac(self):
+        """
+        Test parse_configure_ac with changing () depths and package
+        requirements
+        """
+        conf = config.Config("")
+        content = 'AC_CHECK_FUNC([tgetent])\n'                   \
+                  'XDT_CHECK_PACKAGE(prefix, '                   \
+                  '[module = 2 module2 > 9], '                   \
+                  'action-if-found, action-if-not-found)\n'      \
+                  'next two lines should be read in batch ( \n'  \
+                  'PROG_INTLTOOL\n'                              \
+                  'AC_PROG_SED)\n'                               \
+                  'GETTEXT_PACKAGE'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_configure_ac(os.path.join(tmpd, 'fname'), conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['gettext',
+                              'ncurses-devel',
+                              'perl(XML::Parser)',
+                              'pkgconfig(module2)',
+                              'pkgconfig(module)',
+                              'intltool',
+                              'sed']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_clean_python_req(self):
+        """
+        Test clean_python_req with a common python requirements string
+        """
+        self.assertEqual(buildreq.clean_python_req('requirement >= 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement ; python_version > 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement ; python_version < 1.1.2'),
+                         '')
+        self.assertEqual(buildreq.clean_python_req('requirement <= 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement = 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement \n ; rsa>= 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement != 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('[:python > 2]'),
+                         '')
+        self.assertEqual(buildreq.clean_python_req('requirement ~= 1.1.2'),
+                         'requirement')
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_clean_python_req_comment(self):
+        """
+        Test clean_python_req with a comment
+        """
+        self.assertEqual(buildreq.clean_python_req('# hello'), '')
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_clean_python_req_whitespace(self):
+        """
+        Test clean_python_req with strange whitespaced string
+        """
+        self.assertEqual(buildreq.clean_python_req('   requirement    < 1.1'),
+                        'requirement')
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_grab_python_requirements(self):
+        """
+        Test grab_python_requirements with a reasonable requirements file
+        """
+        # buildreqs must include the requires also
+        open_name = 'buildreq.util.open_auto'
+        content = 'req1 <= 1.2.3\n' \
+                  'req2 >= 1.55\n'  \
+                  'req7 == 3.3.3\n'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.grab_python_requirements('filename', [])
+
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_grab_python_requirements_strange_file(self):
+        """
+        Test grab_python_requirements with a poorly written file
+        """
+        # buildreqs must include the requires also
+        open_name = 'buildreq.util.open_auto'
+        content = '    req1 <= 1.2.3\n   ' \
+                  'req2    >= 1.55   \n'   \
+                  '   req7 == 3.3.3\n    '
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.grab_python_requirements('filename', [])
+
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires(self):
+        """
+        Test add_setup_py_requires with a single item in install_requires and
+        setup_requires
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=['req1']\n" \
+                  "setup_requires=['req2']"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline(self):
+        """
+        Test add_setup_py_requires with a multiline item in install_requires
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=['req1',\n" \
+                  "'req2',\n"                   \
+                  "'req7']\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline_formatted(self):
+        """
+        Test add_setup_py_requires with a multiline item in install_requires
+        with brackets on their own lines.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[\n "  \
+                  "'req1',\n"              \
+                  "'req2',\n"              \
+                  "'req7',\n"              \
+                  "]\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline_variable(self):
+        """
+        Test add_setup_py_requires with multiline item in install_requires that
+        contains a non-literal object.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[\n" \
+                  "reqvar,\n"            \
+                  "'req1',\n"            \
+                  "'req2'"               \
+                  "]\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline_install_requires_variable(self):
+        """
+        Test add_setup_py_requires with multiline item in install_requires that
+        contains an extra bit of content that shouldn't be parsed as install_requires.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[\n"   \
+                  "'req1',\n"              \
+                  "'req2'"                 \
+                  "] + install_requires\n" \
+                  "'bad']\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_variable(self):
+        """
+        Test add_setup_py_requires that contains a non-literal object.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[reqname, 'req1', 'req2']\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_single_variable(self):
+        """
+        Test add_setup_py_requires with a single non-literal object
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires='reqname'"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(reqname)']))
+        self.assertEqual(self.reqs.requires['python3'], set(['pypi(reqname)']))
+
+    def test_scan_for_configure_setup(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'setup.py'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-distutils3']))
+
+    def test_scan_for_configure_cmake(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'CMakeLists.txt'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-cmake']))
+
+    def test_scan_for_configure_scons(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'SConstruct'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-scons']))
+
+    def test_scan_for_configure_meson(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        conf.config_opts['use_ninja'] = False
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'meson.build'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-meson']))
+
+    def test_scan_for_configure_pypi(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test pypi metadata handling.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "name"
+        requires = ["abc",
+                    "def"]
+        pypi_requires = set(f"pypi({x})" for x in requires)
+        summary = "summary"
+        content = json.dumps({"name": name,
+                              "summary": summary,
+                              "requires": requires})
+        buildreq.pypidata.get_pypi_name = MagicMock(return_value=True)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
+
+        ssummary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, name)
+        self.assertEqual(self.reqs.requires['python3'], pypi_requires)
+        self.assertEqual(ssummary, summary)
+
+    def test_scan_for_configure_pypi_no_name_in_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test connecting to pypi but not getting a name back.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "pypi-name"
+        content = json.dumps({})
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_no_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test being unable to connect to pypi.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "pypi-name"
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value="")
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_bad_json(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test being given bad json data.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        orig_json_loads = buildreq.json.loads
+        name = "pypi-name"
+        content = json.dumps({})
+        buildreq.pypidata.pkg_search = MagicMock(return_value=False)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        buildreq.json.loads = MagicMock(side_effect=json.JSONDecodeError("", "", 0))
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), name, conf)
+
+        post_summary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+        buildreq.json.loads = orig_json_loads
+
+        self.assertEqual(self.reqs.pypi_provides, "name")
+        self.assertEqual(post_summary, orig_summary)
+
+    def test_scan_for_configure_pypi_override(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test pypi metadata file override handling.
+        """
+        open_name = 'buildreq.open'
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        name = "name"
+        summary = "summary"
+        requires = ["req"]
+        pypi_requires = set(f"pypi({x})" for x in requires)
+        content = json.dumps({"name": name,
+                              "summary": summary,
+                              "requires": requires})
+        m_open = mock_open(read_data=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            open(os.path.join(tmpd, 'pypi.json'), 'w').close()
+            with patch(open_name, m_open, create=True):
+                self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
+
+        ssummary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+
+        self.assertEqual(self.reqs.pypi_provides, name)
+        self.assertEqual(self.reqs.requires['python3'], pypi_requires)
+        self.assertEqual(ssummary, summary)
+
+    def test_scan_for_configure_setup_with_requires(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test requires.txt handling.
+        """
+        self.reqs.add_pyproject_requires = MagicMock()
+        self.reqs.add_setup_py_requires = MagicMock()
+        self.reqs.grab_python_requirements = MagicMock()
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.config_opts['use_ninja'] = False
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
+            open(os.path.join(tmpd, 'subdir', 'requires.txt'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
+
+        self.reqs.add_pyproject_requires.assert_not_called()
+        self.reqs.add_setup_py_requires.assert_called_once()
+        self.reqs.grab_python_requirements.assert_called_once()
+
+    def test_scan_for_configure_ninja(self):
+        """
+        Test scan_for_configure when ninja is enabled.
+        """
+        conf = config.Config("")
+        conf.config_opts['use_ninja'] = True
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'setup.py'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-distutils3', 'ninja']))
+
+    def test_parse_cmake_pkg_check_modules(self):
+        """
+        Test parse_cmake to ensure accurate detection of versioned and
+        unversioned pkgconfig modules.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = 'pkg_check_modules(GLIB gio-unix-2.0>=2.46.0 glib-2.0 REQUIRED)'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)']))
+
+    def test_parse_cmake_pkg_check_modules_whitespace(self):
+        """
+        Test parse_cmake to ensure accurate handling of versioned
+        pkgconfig modules with whitespace.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = 'pkg_check_modules(GLIB gio-unix-2.0 >= 2.46.0 glib-2.0 REQUIRED)'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)']))
+
+    def test_parse_cmake_pkg_check_modules_in_a_comment(self):
+        """
+        Test parse_cmake to ensure it ignores pkg_check_modules in comments.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = '''
+# For example, consider the following patch to some CMakeLists.txt.
+#     - pkg_check_modules(FOO REQUIRED foo>=1.0)
+#     + pkg_check_modules(FOO REQUIRED foo>=2.0)
+'''
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set([]))
+
+    def test_parse_cmake_pkg_check_modules_variables(self):
+        """
+        Test parse_cmake to ensure accurate handling of versioned
+        pkgconfig modules with variable version strings.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = 'pkg_check_modules(AVCODEC libavcodec${_avcodec_ver} libavutil$_avutil_ver)'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)']))
+
+    def test_parse_cmake_find_package(self):
+        """
+        Test parse_cmake to ensure accurate handling of find_package.
+        """
+        cmake_modules = {
+            "valid": "valid",
+            "valid_but_commented": "valid_but_commented",
+            "different_name": "another_name",
+            "qt6.module1": "qt6module1",
+            "qt6.module2": "qt6module2",
+            "kf6.module3": "kf6module3",
+            "kf6.module4": "kf6module4",
+            ".module5": "namodule5",
+            ".module6": "namodule6"
+        }
+        content = '''
+find_package(valid)
+#find_package(foo)
+  # find_package(valid_but_commented)
+find_package(different_name)
+find_package(Qt6 stuff
+module1
+module2)
+find_package(KF6 stuff
+module3
+module4
+)
+find_package(NOT_HANDLED_NAMESPACE stuff
+module5
+
+module6
+)
+'''
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['valid',
+                              'another_name',
+                              'qt6module1',
+                              'qt6module2',
+                              'kf6module3',
+                              'kf6module4',
+                              'namodule5',
+                              'namodule6']))
+
+    def test_r_desc_field_begin(self):
+        """Test parsing of the first R description field."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, ["foo"])
+
+    def test_r_desc_field_middle(self):
+        """Test parsing of an R description field with surrounding fields."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, ["bar"])
+
+    def test_r_desc_field_end(self):
+        """Test parsing of the last R description field."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field3", "\n".join(lines))
+        self.assertEqual(result, ["baz"])
+
+    def test_r_desc_field_middle_multiple_lines(self):
+        """Test parsing of a multi-line R description field, with surrounding fields."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar1, bar2,",
+            "  bar3, bar4",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, ["bar1", "bar2", "bar3", "bar4"])
+
+    def test_r_desc_field_end_multiple_lines(self):
+        """Test parsing of the last R description field, consisting of multiple lines."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz1,",
+            "        baz2",
+        ]
+        result = buildreq._get_desc_field("Field3", "\n".join(lines))
+        self.assertEqual(result, ["baz1", "baz2"])
+
+    def test_r_desc_field_middle_one_per_line(self):
+        """Test parsing of an R description field with one entry per line."""
+        lines = [
+            "Field1: foo",
+            "Field2:",
+            "bar1,",
+            "bar2",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, ["bar1", "bar2"])
+
+    def test_r_desc_field_trailing_whitespace(self):
+        """Test parsing of an R description field with trailing whitespace."""
+        lines = [
+            "Field1: foo1, foo2    ",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, ["foo1", "foo2"])
+
+    def test_r_desc_field_trailing_comma(self):
+        """Test parsing of an R description field with trailing comma."""
+        lines = [
+            "Field1: foo1, foo2,",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, ["foo1", "foo2"])
+
+    def test_r_desc_field_empty(self):
+        """Test parsing of an R description field with an empty value."""
+        lines = [
+            "Field1:",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, [])
+
+    def test_r_desc_field_missing(self):
+        """Test parsing of an R description field that is missing."""
+        lines = [
+            "Field1: foo, bar",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, [])
+
+    def test_parse_r_desc_depends(self):
+        """Test parsing of R description Depends field."""
+        pkgs = ['R-pkg1']
+        open_name = 'buildreq.util.open_auto'
+        content = 'Depends: pkg1'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg1' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_imports(self):
+        """Test parsing of an R description Imports field."""
+        pkgs = ['R-pkg2']
+        open_name = 'buildreq.util.open_auto'
+        content = 'Imports: pkg2'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg2' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_linkingto(self):
+        """Test parsing of an R description LinkingTo field."""
+        pkgs = ['R-pkg3']
+        open_name = 'buildreq.util.open_auto'
+        content = 'LinkingTo: pkg3'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg3' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_multiple(self):
+        """Test parsing of an R description file that captures multiple fields."""
+        pkgs = [
+            'R-pkg1',
+            'R-pkg2',
+            'R-pkg3',
+            'R-pkg4',
+        ]
+        open_name = 'buildreq.util.open_auto'
+        content = [
+            'Field1: foo',
+            'Imports: pkg1, pkg2,',
+            '            pkg3   ',
+            'LinkingTo: pkg4',
+            'FieldFoo: bar',
+        ]
+        m_open = mock_open(read_data='\n'.join(content))
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertFalse('R-foo' in self.reqs.buildreqs)
+        self.assertFalse('R-bar' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg1' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg2' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg3' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg4' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_not_in_os(self):
+        """Test parsing of an R description file with some non-OS packages."""
+        pkgs = [
+            'R-pkg1',
+        ]
+        open_name = 'buildreq.util.open_auto'
+        content = [
+            'Imports: pkg1, pkg2',
+            'Depends: pkg3',
+        ]
+        m_open = mock_open(read_data='\n'.join(content))
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg1' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg1' in self.reqs.requires[None])
+        # Names absent from the os-packages list are also added, because the
+        # DESCRIPTION file is considered authoritative.
+        for pkg in ['R-pkg2', 'R-pkg3']:
+            self.assertTrue(pkg in self.reqs.buildreqs)
+            self.assertTrue(pkg in self.reqs.requires[None])
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_check.py

@@ -0,0 +1,195 @@
+import os
+import tempfile
+import unittest
+from unittest.mock import mock_open, patch
+
+import buildreq
+import check
+import config
+import tarball
+
+def mock_generator(rv=None):
+    def mock_f(*args, **kwargs):
+        return rv
+
+    return mock_f
+
+
+class TestTest(unittest.TestCase):
+    backup_isfile = check.os.path.isfile
+
+    @classmethod
+    def setUpClass(self):
+        self.open_name = 'check.util.open_auto'
+        check.os.path.isfile = mock_generator(True)
+
+    @classmethod
+    def tearDownClass(self):
+        check.os.path.isfile = self.backup_isfile
+
+    def setUp(self):
+        check.tests_config = ''
+
+    def test_check_regression(self):
+        """
+        Test check_regression
+        """
+        def mock_parse_log(log):
+            return ',120,100,20,0,0'
+
+        parse_log_backup = check.count.parse_log
+        check.count.parse_log = mock_parse_log
+        m_open = mock_open()
+        open_name = 'util.open'
+        with patch(open_name, m_open, create=True):
+            check.check_regression('pkgdir', False, -1)
+
+        check.count.parse_log = parse_log_backup
+
+        exp_call = unittest.mock.call().write('Total : 120\n'
+                                              'Pass : 100\n'
+                                              'Fail : 20\n'
+                                              'Skip : 0\n'
+                                              'XFail : 0\n')
+        self.assertIn(exp_call, m_open.mock_calls)
+
+    def test_check_regression_multi(self):
+        """
+        Test check_regression with multiple results
+        """
+        def mock_parse_log(log):
+            return 'test-a,120,100,20,0,0\ntest-b,10,5,3,2,1'
+
+        parse_log_backup = check.count.parse_log
+        check.count.parse_log = mock_parse_log
+        m_open = mock_open()
+        open_name = 'util.open'
+        with patch(open_name, m_open, create=True):
+            check.check_regression('pkgdir', False, -1)
+
+        check.count.parse_log = parse_log_backup
+
+        exp_call = unittest.mock.call().write('Package : test-a\n'
+                                              'Total : 120\n'
+                                              'Pass : 100\n'
+                                              'Fail : 20\n'
+                                              'Skip : 0\n'
+                                              'XFail : 0\n'
+                                              'Package : test-b\n'
+                                              'Total : 10\n'
+                                              'Pass : 5\n'
+                                              'Fail : 3\n'
+                                              'Skip : 2\n'
+                                              'XFail : 1\n')
+        self.assertIn(exp_call, m_open.mock_calls)
+
+    def test_scan_for_tests_makecheck_in(self):
+        """
+        Test scan_for_tests with makecheck suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.in'])
+        content = 'check:'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "configure"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'make %{?_smp_mflags} check')
+
+    def test_scan_for_tests_makecheck_am(self):
+        """
+        Test scan_for_tests with makecheck suite via Makefile.am
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.am'])
+        m_open = mock_open()
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "configure_ac"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'make %{?_smp_mflags} check')
+
+    def test_scan_for_tests_perlcheck_PL(self):
+        """
+        Test scan_for_tests with perlcheck suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.PL'])
+        conf.default_pattern = "cpan"
+        check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test')
+
+    def test_scan_for_tests_perlcheck_in(self):
+        """
+        Test scan_for_tests with perlcheck suite via Makefile.in
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.in'])
+        content = 'test:'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "cpan"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test')
+
+    def test_scan_for_tests_cmake(self):
+        """
+        Test scan_for_tests with cmake suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['CMakeLists.txt'])
+        content = 'enable_testing'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "cmake"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'cd clr-build; make test')
+
+    def test_scan_for_tests_tox_requires(self):
+        """
+        Test scan_for_tests with tox.ini in the files list, should add several
+        build requirements
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['tox.ini'])
+        check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+        check.os.listdir = listdir_backup
+        self.assertEqual(reqs.buildreqs,
+                         set(['pypi-tox',
+                              'pypi-pytest',
+                              'pypi-virtualenv',
+                              'pypi-pluggy',
+                              'pypi(py)']))
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=True)

tests/test_commitmessage.py

@@ -0,0 +1,245 @@
+import unittest
+import unittest.mock as mock
+import os
+import tempfile
+import build
+import commitmessage
+import config
+import tarball
+
+
+class TestCommitmessage(unittest.TestCase):
+
+    def setUp(self):
+        self.workingdir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.workingdir.cleanup()
+
+    def test_is_header(self):
+        """
+        Test is_header function with list of lines. First and last line, line
+        followed by a line containing '---', and lines after a blank '' line
+        should be recognized as headers. Last line recognized as header because
+        it is a relevant ending point.
+        """
+        lines = ['line1',  # True
+                 'line2',  # False
+                 '',       # False
+                 'line4',  # True
+                 'line5',  # True
+                 '---',    # False
+                 'line7',  # False
+                 'line8']  # True
+        for idx, line in enumerate(lines):
+            print(line, idx)
+            if idx in [0, 3, 4, 7]:
+                self.assertTrue(commitmessage.is_header(lines, idx))
+            else:
+                self.assertFalse(commitmessage.is_header(lines, idx))
+
+    def test_find_in_line(self):
+        """
+        Trivially tests commitmessage.find_in_line(). Just makes sure results
+        evaluate to the correct bool.
+        """
+        self.assertTrue(commitmessage.find_in_line(r'Version', 'AVersionInThisLine'))
+        self.assertFalse(commitmessage.find_in_line(r'z', 'the quick brown fox jumps over the lady dog'))
+
+    def test_process_NEWS(self):
+        """
+        Test process_NEWS() function with valid newsfile provided
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                newsfile.write(GOOD_NEWS)
+            # commitmessage returned will have an empty string as first and
+            # last items
+            expected_msg = [""] + GOOD_NEWS.split('\n')[3:13]
+            expected_cvs = set()
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd),
+                             (expected_msg, expected_cvs))
+
+    def test_process_NEWS_bad_news(self):
+        """
+        Test process_NEWS() function with irrelevant newsfile provided
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                # make GOOD_NEWS irrelevant by replacing current version
+                newsfile.write(GOOD_NEWS.replace('0.0.1', '0.0.0'))
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd), ([], set()))
+
+    def test_process_NEWS_good_cves(self):
+        """
+        Test process_NEWS() function with valid newsfile and CVEs
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                # give GOOD_NEWS some CVEs
+                newsfile.write(GOOD_NEWS.replace('change2.1', 'CVE-2-1')
+                                        .replace('change2.2', 'CVE-2-2'))
+            # commitmessage returned will have an empty string as first and
+            # last items.
+            # replace change2.* strings with CVE strings
+            expected_msg = [""] + GOOD_NEWS.replace('change2.1', 'CVE-2-1')\
+                                           .replace('change2.2', 'CVE-2-2')\
+                                           .split('\n')[3:13]
+            expected_cvs = set(['CVE-2-1', 'CVE-2-2'])
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd),
+                             (expected_msg, expected_cvs))
+
+    def test_process_NEWS_long(self):
+        """
+        Test process_NEWS() function with valid newsfile provided, but relevant
+        block is longer than 15 lines, causing it to be truncated.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            long_news = GOOD_NEWS.replace('text explaining change2.2',
+                                          '1\n2\n3\n4\n5\n6\n7\n8\n9\n')
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                newsfile.write(long_news)
+            # commitmessage returned will have an empty string as first and
+            # last items, extend the expected message with extra lines and
+            # truncate message.
+            expected_msg = [""] + GOOD_NEWS.split('\n')[3:11]
+            expected_msg.extend(['1', '2', '3', '4', '5', '6', '7', '',
+                                 '(NEWS truncated at 15 lines)', ''])
+            expected_cvs = set()
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd),
+                             (expected_msg, expected_cvs))
+
+    def test_guess_commit_message(self):
+        """
+        Test guess_commit_message() with mocked internal functions and both
+        commitmessage information and cves available from newsfile.
+        """
+        conf = config.Config("")
+        conf.old_version = "0.0.0"
+        tcontent = tarball.Content("", "testball", "0.0.1", [], conf, self.workingdir.name)
+        conf.content = tcontent
+        process_NEWS_backup = commitmessage.process_NEWS
+
+        def mock_process_NEWS(newsfile, old_version, name, version, download_path):
+            return (['', 'commit', 'message', 'with', 'cves', ''],
+                    set(['cve1', 'cve2']))
+
+        commitmessage.process_NEWS = mock_process_NEWS
+        open_name = 'util.open_auto'
+        with mock.patch(open_name, create=True) as mock_open:
+            mock_open.return_value = mock.MagicMock()
+            conf.rewrite_config_opts = mock.Mock()
+            commitmessage.guess_commit_message("", conf, tcontent)
+            # reset mocks before asserting so a failure doesn't cascade to
+            # other tests
+            commitmessage.process_NEWS = process_NEWS_backup
+            fh = mock_open.return_value.__enter__.return_value
+            fh.write.assert_called_with(
+                'testball: Autospec creation for update from version 0.0.0 to '
+                'version 0.0.1\n\n\ncommit\nmessage\nwith\ncves\n\n\ncommit\n'
+                'message\nwith\ncves\n\nCVEs fixed in this build:\ncve1\ncve2'
+                '\n\n')
+
+    def test_guess_commit_message_cve_config(self):
+        """
+        Test guess_commit_message() with mocked internal functions and both
+        commitmessage information and cves available from newsfile. A cve is
+        also available from config, which changes the first line of the commmit
+        message.
+        """
+        conf = config.Config("")
+        tcontent = tarball.Content("", "testball", "0.0.1", [], conf, self.workingdir.name)
+        conf.content = tcontent
+        process_NEWS_backup = commitmessage.process_NEWS
+
+        def mock_process_NEWS(newsfile, old_version, name, version, download_path):
+            return (['', 'commit', 'message', 'with', 'cves', ''],
+                    set(['cve1', 'cve2']))
+
+        commitmessage.process_NEWS = mock_process_NEWS
+        conf.cves = set(['CVE-1234-5678'])
+        conf.old_version = None  # Allow cve title to be set
+        open_name = 'util.open_auto'
+        with mock.patch(open_name, create=True) as mock_open:
+            mock_open.return_value = mock.MagicMock()
+            conf.rewrite_config_opts = mock.Mock()
+            commitmessage.guess_commit_message("", conf, tcontent)
+            # reset mocks before asserting so a failure doesn't cascade to
+            # other tests
+            commitmessage.process_NEWS = process_NEWS_backup
+            fh = mock_open.return_value.__enter__.return_value
+            fh.write.assert_called_with(
+                'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n'
+                'cves\n\n\ncommit\nmessage\nwith\ncves\n\nCVEs fixed in this '
+                'build:\nCVE-1234-5678\ncve1\ncve2\n\n')
+
+    def test_guess_commit_message_imported_key(self):
+        """
+        Test guess_commit_message() with mocked internal functions and both
+        commitmessage information and cves available from newsfile. A cve is
+        also available from config, which changes the first line of the commmit
+        message. Additionally there is imported key info that will be displayed
+        at the end of the message.
+        """
+        conf = config.Config("")
+        tcontent = tarball.Content("", "testball", "0.0.1", [], conf, self.workingdir.name)
+        conf.content = tcontent
+        process_NEWS_backup = commitmessage.process_NEWS
+
+        def mock_process_NEWS(newsfile, old_version, name, version, download_path):
+            return (['', 'commit', 'message', 'with', 'cves', ''],
+                    set(['cve1', 'cve2']))
+
+        commitmessage.process_NEWS = mock_process_NEWS
+        conf.cves = set(['CVE-1234-5678'])
+        conf.old_version = None  # Allow cve title to be set
+        open_name = 'util.open_auto'
+        with mock.patch(open_name, create=True) as mock_open:
+            mock_open.return_value = mock.MagicMock()
+            conf.rewrite_config_opts = mock.Mock()
+            commitmessage.guess_commit_message("keyinfo content", conf, tcontent)
+            # reset mocks before asserting so a failure doesn't cascade to
+            # other tests
+            commitmessage.process_NEWS = process_NEWS_backup
+            fh = mock_open.return_value.__enter__.return_value
+            fh.write.assert_called_with(
+                'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n'
+                'cves\n\n\ncommit\nmessage\nwith\ncves\n\nCVEs fixed in this '
+                'build:\nCVE-1234-5678\ncve1\ncve2\n\nKey imported:\nkeyinfo '
+                'content\n')
+
+    def test_scan_for_changes(self):
+        """
+        Tests scan_for_changes using temporary directories
+        """
+        conf = config.Config("")
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'changelog.txt'), 'w') as newsfile:
+                newsfile.write('new changelog file')
+
+            with tempfile.TemporaryDirectory() as tmpd1:
+                commitmessage.scan_for_changes(tmpd1, tmpd, conf.transforms)
+                self.assertTrue(os.path.isfile(tmpd1 + '/ChangeLog'))
+
+
+GOOD_NEWS = """
+GOOD NEWS -- History of user-visible changes.
+
+* Version 0.0.1
+
+change1.1
+change1.2
+
+change2.1
+text explaining change2.1
+change2.2
+text explaining change2.2
+
+* Version 0.0.0
+
+This better not show up
+"""
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_config.py

@@ -0,0 +1,90 @@
+import unittest
+import config
+
+
+# Input for tarball.detect_build_from_url method tests
+# Structure: (url, build_pattern)
+BUILD_PAT_URL = [
+    ("https://cran.r-project.org/src/contrib/raster_3.0-12.tar.gz", "R"),
+    ("https://ftp.osuosl.org/pub/cran/src/contrib/hexbin_1.28.5.tar.gz", "R"),
+    ("http://pypi.debian.net/argparse/argparse-1.4.0.tar.gz", "distutils3"),
+    ("https://pypi.python.org/packages/source/T/Tempita/Tempita-0.5.2.tar.gz", "distutils3"),
+    ("https://cpan.metacpan.org/authors/id/T/TO/TODDR/IO-Tty-1.14.tar.gz", "cpan"),
+    ("http://search.cpan.org/CPAN/authors/id/D/DS/DSKOLL/IO-stringy-2.111.tar.gz", "cpan"),
+    ("https://pecl.php.net//get/lua-2.0.6.tgz", "phpize"),
+]
+
+
+def detect_build_test_generator(url, build_pattern):
+    """Create test for tarball.detect_build_from_url method."""
+    def generator(self):
+        """Test template."""
+        conf = config.Config("")
+        conf.detect_build_from_url(url)
+        self.assertEqual(build_pattern, conf.default_pattern)
+
+    return generator
+
+
+def create_dynamic_tests():
+    # Create tests for config.detect_build_from_url method.
+    for url, build_pattern in BUILD_PAT_URL:
+        test_name = 'test_pat_{}'.format(url)
+        test = detect_build_test_generator(url, build_pattern)
+        setattr(TestConfig, test_name, test)
+
+
+class TestConfig(unittest.TestCase):
+
+    def test_set_build_pattern(self):
+        """
+        Test set_build_pattern with sufficient pattern strength
+        """
+        conf = config.Config("")
+        conf.set_build_pattern("configure_ac", 1)
+        self.assertEqual(conf.default_pattern, "configure_ac")
+        self.assertEqual(conf.pattern_strength, 1)
+
+    def test_set_build_pattern_low_strength(self):
+        """
+        Test set_build_pattern with low pattern strength, nothing in the module
+        should change
+        """
+        conf = config.Config("")
+        conf.pattern_strength = 2
+        conf.set_build_pattern("configure_ac", 1)
+        self.assertEqual(conf.default_pattern, "make")
+        self.assertEqual(conf.pattern_strength, 2)
+
+    def test_validate_extras_content_bad_glob(self):
+        """
+        Test validate_extras_content with more than one glob in a directory
+        """
+        conf = config.Config("")
+        lines = ['/bad*path*/file']
+        new_lines = conf.validate_extras_content(lines, 'bad_glob')
+        self.assertEqual(len(new_lines), 0)
+
+    def test_validate_extras_content_good_single_glob(self):
+        """
+        Test validate_extras_content with a single glob in a directory
+        """
+        conf = config.Config("")
+        lines = ['/good*/file']
+        new_lines = conf.validate_extras_content(lines, 'good_single_glob')
+        self.assertEqual(new_lines, [lines[0].split('/')])
+
+    def test_validate_extras_content_good_multi_glob(self):
+        """
+        Test validate_extras_content with a multiple valid globs
+        """
+        conf = config.Config("")
+        lines = ['/path1', '/good*/glob*/file', '/path3']
+        new_lines = conf.validate_extras_content(lines, 'good_multi_glob')
+        self.assertEqual(new_lines, ['/path1', lines[1].split('/'), '/path3'])
+
+# Create dynamic tests
+create_dynamic_tests()
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_count.py

@@ -0,0 +1,555 @@
+import unittest
+from unittest.mock import mock_open, patch
+import count
+
+pats = [
+    # acl
+    ('[22] $ rm -Rf d -- ok-',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('17 commands (17 passed, 1 failed)-',
+     [18, 17, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # alembic
+    ('Ran 678 tests in 5.175s',
+     [678, 678, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('OK (SKIP=15)',
+     [15, 0, 0, 0, 15, 0, 0, 0, 0, 0]),
+    ('OK (skipped=16)',
+     [16, 0, 0, 0, 16, 0, 0, 0, 0, 0]),
+    # anyjson
+    ('test_implementations.test_default_serialization ... ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('test_implementations.test_default_serialization ... skipped',
+     [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]),
+    # apr
+    ('testatomic          :  SUCCESS',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # cryptography
+    ('================= 76230 passed, 267 skipped in 140.23 seconds ==================',
+     [76497, 76230, 0, 0, 267, 0, 0, 0, 0, 0]),
+    ('================== 47 passed, 2 error in 10.36 seconds =========================',
+     [49, 47, 2, 0, 0, 0, 0, 0, 0, 0]),
+    ('================ 10 failed, 16 passed, 4 error in 0.16 seconds =================',
+     [30, 16, 14, 0, 0, 0, 0, 0, 0, 0]),
+    ('========================== 43 passed in 2.90 seconds ===========================',
+     [43, 43, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('========================== 43 passed in 2.90s ===========================',
+     [43, 43, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('======= 28 failed, 281 passed, 13 skipped, 10 warnings in 28.48 seconds ========',
+     [332, 281, 38, 0, 13, 0, 0, 0, 0, 0]),
+    ('===================== 5 failed, 318 passed in 1.06 seconds =====================',
+     [323, 318, 5, 0, 0, 0, 0, 0, 0, 0]),
+    ('===================== 5 failed, 9 passed, 7 xfailed in 1.06 seconds ============',
+     [21, 9, 5, 7, 0, 0, 0, 0, 0, 0]),
+    ('============= 1628 passed, 72 skipped, 4 xfailed in 146.26 seconds =============',
+     [1704, 1628, 0, 4, 72, 0, 0, 0, 0, 0]),
+    ('=============== 119 passed, 2 skipped, 54 error in 2.19 seconds ================',
+     [175, 119, 54, 0, 2, 0, 0, 0, 0, 0]),
+    ('========== 1 failed, 74 passed, 10 skipped, 55 error in 2.05 seconds ===========',
+     [140, 74, 56, 0, 10, 0, 0, 0, 0, 0]),
+    ('==================== 68 passed, 1 warnings in 0.12 seconds =====================',
+     [69, 68, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('================ 3 failed, 250 passed, 3 error in 3.28 seconds =================',
+     [256, 250, 6, 0, 0, 0, 0, 0, 0, 0]),
+    ('=============== 1 failed, 407 passed, 10 skipped in 4.71 seconds ===============',
+     [418, 407, 1, 0, 10, 0, 0, 0, 0, 0]),
+    ('=============== 1 failed, 407 passed, 10 skipped in 4.71s ===============',
+     [418, 407, 1, 0, 10, 0, 0, 0, 0, 0]),
+    ('========================== 1 skipped in 0.79 seconds ===========================',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('========================== 1 skipped in 0.79s ===========================',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('=========================== 3 error in 0.41 seconds ============================',
+     [3, 0, 3, 0, 0, 0, 0, 0, 0, 0]),
+    ('=========================== 3 error in 0.41s ============================',
+     [3, 0, 3, 0, 0, 0, 0, 0, 0, 0]),
+    ('================= 68 passed, 1 pytest-warnings in 0.09 seconds =================',
+     [68, 68, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('================= 68 passed, 1 pytest-warnings in 0.09s =================',
+     [68, 68, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('===== 21 failed, 73 passed, 5 skipped, 2 pytest-warnings in 34.81 seconds ======',
+     [99, 73, 21, 0, 5, 0, 0, 0, 0, 0]),
+    ('===== 21 failed, 73 passed, 5 skipped, 2 pytest-warnings in 34.81s ======',
+     [99, 73, 21, 0, 5, 0, 0, 0, 0, 0]),
+    # swift
+    ('========= 1 failed, 1287 passed, 1 warnings, 62 error in 35.77 seconds =========',
+     [1351, 1287, 64, 0, 0, 0, 0, 0, 0, 0]),
+    ('===== 487 failed, 4114 passed, 32 skipped, 1 pytest-warnings, 34 error in 222.82 seconds =====',
+     [4667, 4114, 521, 0, 32, 0, 0, 0, 0, 0]),
+    # tox
+    ('======== 199 passed, 38 skipped, 1 xpassed, 1 warnings in 5.76 seconds =========',
+     [239, 200, 1, 0, 38, 0, 0, 0, 0, 0]),
+    # augeas
+    ('# TOTAL: 215\n'
+     '# PASS:  212\n'
+     '# SKIP:  3\n'
+     '# XFAIL: 0\n'
+     '# FAIL:  0\n'
+     '# XPASS: 0\n'
+     '# ERROR: 0',
+     [215, 212, 0, 0, 3, 0, 0, 0, 0, 0]),
+    # autoconf
+    ('493 tests behaved as expected\n'
+     '10 tests were skipped.\n'
+     '495: AC_FUNC_STRNLEN                                 ok\n'
+     '344: Erlang                                          skipped (erlang.at:30)\n'
+     '26: autoupdating macros recursively                 expected failure (tools.at:945)',
+    [503, 493, 0, 0, 10, 3, 1, 0, 1, 1]),
+    # bison
+    ('470 tests were successful',
+    [470, 470, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # binutils
+    ('# of expected passes\t1144\n'
+     '# of expected failures\t57\n'
+     '# of untested testcases\t1\n'
+     '# of unsupported tests\t12\n'
+     '# of unexpected failures\t1\n',
+     [1214, 1144, 1, 57, 12, 0, 0, 0, 0, 0]),
+    # ccache
+    ('PASSED: 448 assertions, 88 tests, 10 suites',
+     [88, 88, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # rubygem-rack
+    ('701 tests, 2292 assertions, 0 failures, 0 errors',
+     [701, 701, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # curl
+    ('TESTDONE: 680 tests out of 686 reported OK: 99%',
+     [686, 680, 6, 0, 0, 0, 0, 0, 0, 0]),
+    # gcc
+    ('All 4 tests passed\n'
+     'PASS: test-strtol-16\n'
+     'FAIL: test-strtol-32',
+     [4, 4, 0, 0, 0, 2, 1, 1, 0, 0]),
+    # gdbm
+    ('All 22 tests were successful.',
+     [22, 22, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # glibc
+    ('3 FAIL\n'
+     '2182 PASS\n'
+     '1 UNRESOLVED\n'
+     '199 XFAIL\n'
+     '3 XPASS',
+     [2387, 2185, 3, 199, 0, 0, 0, 0, 0, 0]),
+    # libxml2
+    ('Total 2908 tests, no errors',
+     [2908, 2908, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('Total: 1171 functions, 291083 tests, 0 errors',
+     [1171, 1171, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # zlib
+    ('*** zlib shared test OK ***',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # e2fsprogs
+    ('153 tests succeeded     1 tests failed',
+     [154, 153, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # expect
+    ('all.tcl:     Total   41     Passed   29     Skipped   2     Failed   10',
+     [41, 29, 10, 0, 2, 0, 0, 0, 0, 0]),
+    # expat
+    ('50%: Checks: 50, Failed: 25',
+     [50, 25, 25, 0, 0, 0, 0, 0, 0, 0]),
+    # flex
+    ('Tests succeeded: 47\n'
+     'Tests FAILED: 3',
+     [50, 47, 3, 0, 0, 0, 0, 0, 0, 0]),
+    # TAP and perl-Capture-tiny
+    ('ok 580 - tee_merged|sys|stderr|short = got STDERR',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('not ok 580 - tee_merged|sys|stderr|short = got STDERR',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    ('not ok 580 - tee_merged|sys|stderr|short = got STDERR  # TODO known breakage',
+     [0, 0, 0, 0, 0, 0, 0, 0, 1, 0]),
+    ('ok 9',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('not ok 9',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # tcpdump
+    ('    1 test failed\n'
+     ' 154 tests passed',
+     [155, 154, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('    2 tests failed\n'
+     ' 154 tests passed',
+     [156, 154, 2, 0, 0, 0, 0, 0, 0, 0]),
+    # R packages
+    ('* checking top-level files ... OK',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('* checking top-level files ... PASSED.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('* checking top-level files ... SKIPPED',
+     [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]),
+    # python
+    ('365 tests OK.\n'
+     '22 tests skipped:',
+     [387, 365, 0, 0, 22, 0, 0, 0, 0, 0]),
+    # jemalloc
+    ('Test suite summary: pass: 30/33, skip: 3/33, fail: 0/33',
+     [33, 30, 0, 0, 3, 0, 0, 0, 0, 0]),
+    # util-linux
+    ('  All 160 tests PASSED',
+     [160, 160, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # onig
+    ("OK: // 'a'",
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # nss
+    ('cert.sh: #101: Import chain-2-serverCA-ec CA -t u,u,u for localhost.localdomain (ext.)  - PASSED\n'
+     'Passed:             13036\n'
+     'Failed:             6\n'
+     'Failed with core:   0\n'
+     'Unknown status:     0',
+     [13042, 13036, 6, 0, 0, 1, 1, 0, 0, 0]),
+    # nss
+    ('cert.sh: #101: Import chain-2-serverCA-ec CA -t u,u,u for localhost.localdomain (ext.)  - FAILED\n'
+     'Passed:             13036\n'
+     'Failed:             6\n'
+     'Failed with core:   0\n'
+     'Unknown status:     0',
+     [13042, 13036, 6, 0, 0, 0, 0, 1, 0, 0]),
+    # rsync
+    ('     34 passed\n'
+     '     5 skipped',
+     [39, 34, 0, 0, 5, 0, 0, 0, 0, 0]),
+    # mariadb
+    ('50% tests passed, 20 tests failed out of 40',
+     [40, 20, 20, 0, 0, 0, 0, 0, 0, 0]),
+    # python-runtime-tests
+    ('FAILED (KNOWNFAIL=6, SKIP=18, errors=6)',
+     [30, 0, 6, 6, 18, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1, errors=499, skipped=48)',
+     [548, 0, 1, 499, 48, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1, errors=499)',
+     [500, 0, 1, 499, 0, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1)',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('FAILED (errors=1)',
+     [1, 0, 0, 1, 0, 0, 0, 0, 0, 0]),
+    ('OK (KNOWNFAIL=5, SKIP=15)',
+     [20, 0, 0, 5, 15, 0, 0, 0, 0, 0]),
+    # qpid-python
+    ('Totals: 318 tests, 200 passed, 112 skipped, 0 ignored, 6 failed',
+     [318, 200, 6, 0, 112, 0, 0, 0, 0, 0]),
+    # PyYAML
+    ('TESTS: 2577',
+     [2577, 2577, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # sudo
+    ('visudo: 7/7 tests passed; 0/7 tests failed',
+     [7, 7, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('check_symbols: 7 tests run, 0 errors, 100% success rate',
+     [7, 7, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # R
+    ("running code in 'reg-examples1.R' ... OK",
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Status: 1 ERROR, 1 WARNING, 4 NOTEs',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('OK: 749 SKIPPED: 4 FAILED: 2',
+     [755, 749, 2, 0, 4, 0, 0, 0, 0, 0]),
+    # php
+    ('Number of tests : 13526              9794\n'
+     'Tests skipped   : 3732 ( 27.6%) --------\n'
+     'Tests warned    :    0 (  0.0%) (  0.0%)\n'
+     'Tests failed    :   12 (  0.1%) (  0.1%)\n'
+     'Expected fail   :   31 (  0.2%) (  0.3%)\n'
+     'Tests passed    : 9751 ( 72.1%) ( 99.6%)',
+     [13526, 9751, 12, 31, 3732, 0, 0, 0, 0, 0]),
+    # rubygem/rake
+    ('174 runs, 469 assertions, 0 failures, 0 errors, 0 skips',
+     [174, 174, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # cryptsetup
+    (' [OK]',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # lzo
+    (' test passed.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # lsof
+    ('LTnlink ... OK',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('LTnfs ... ERROR!!!',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # libaio
+    ('Pass: 11  Fail: 1',
+     [12, 11, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # gawk
+    ('ALL TESTS PASSED',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # gptfdisk
+    ('**SUCCESS**',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # boost
+    ('**passed** ...',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('8 errors detected.',
+     [8, 0, 8, 0, 0, 0, 0, 0, 0, 0]),
+    ('8 failures detected.',
+     [8, 0, 8, 0, 0, 0, 0, 0, 0, 0]),
+    # make
+    ('534 Tests in 118 Categories Complete ... No Failures',
+     [534, 534, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # icu4c
+    ('---[OK]',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # libxslt
+    ('Pass 1',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # apr-util
+    (':  SUCCESS',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # bash
+    ('< Failed 126 of 1378 Unicode tests',
+     [1378, 1252, 126, 0, 0, 0, 0, 0, 0, 0]),
+    # crudini
+    ('Test 95 OK (line 460)',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Test 95 BAD',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # discount
+    ('Reddit-style automatic links ............................... OK',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Reddit-style automatic links .............................. BAD',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # libjpeg-turbo
+    ('JPEG -> RGP Top-Down 2/1 ... Passed.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # zlib
+    ('*** zlib test OK ***',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('*** zlib 64-bit test OK ***',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # LVM2
+    ('valgrind pool awareness ... fail',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    ('dfa with non-print regex chars ... pass',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # keyring
+    ('==== 76 passed, 62 skipped, 50 xfailed, 14 xpassed, 2 warnings, 32 error in 2.13 seconds ====',
+     [236, 90, 34, 50, 62, 0, 0, 0, 0, 0]),
+    # openblas
+    ('  ----- PASS -----',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('  ----- FAIL -----',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # rubygem-hashie
+    ('545 examples, 0 failures, 1 pending',
+     [546, 545, 0, 0, 1, 0, 0, 0, 0, 0]),
+    # rubygem-warden
+    ('215 examples, 14 failures',
+     [229, 215, 14, 0, 0, 0, 0, 0, 0, 0]),
+    # rubygem-ansi
+    ('Executed 12 tests with 7 passing, 5 errors.',
+     [12, 7, 5, 0, 0, 0, 0, 0, 0, 0]),
+    # vim
+    ('Executed 12 tests',
+     [12, 12, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # rubygem-formatador
+    ('  9 succeeded in 0.00375661 seconds',
+     [9, 9, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # pigz
+    ('./pigz -kf pigz.c ; ./pigz -t pigz.c.gz',
+     [2, 2, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('./pigz -kfb 32 pigz.c',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # netifaces
+    ('Interface lo:',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # btrfs-progs
+    ('    [TEST]    001-bad-file-extent-bytenr',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('    [NOTRUN]  Need to validate root privileges',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('test failed for case',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # chrpath
+    ('success: chrpath changed rpath to larger path.',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('error: chrpath unable to change rpath to larger path.',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('warning: chrpath does not have root permissions',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # yajl
+    ('58/58 tests successful',
+     [58, 58, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # xmlsec1
+    ('    Checking required transforms                            OK',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('    Verify existing signature                             Fail',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('    Checking required transforms                          Skip',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    # xdg-utils
+    ('TOTAL: 4 tests failed, 90 of 116 tests passed. (140 attempted)',
+    [140, 90, 4, 0, 46, 0, 0, 0, 0, 0]),
+    # slang
+    ('Testing argv processing ...Ok',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('./utf8.sl:14:check_sprintf:Test Error',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # go & golang
+    ('ok    golang.org/x/text/encoding/htmlindex    0.002s',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('--- FAIL: TestParents (0.00s)',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('FAIL	golang.org/x/text/internal	0.002s',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('--- PASS: TestApp_Command (0.00s)',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # valgrind
+    ('== 5 tests, 0 stderr failures, 1 stdout failure, 0 stderrB failures, 0 stdoutB failures, 0 post failures ==',
+     [5, 4, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('== 55 tests, 48 stderr failures, 6 stdout failures, 0 stderrB failures, 0 stdoutB failures, 0 post failures ==',
+     [55, 1, 54, 0, 0, 0, 0, 0, 0, 0]),
+    ('== 125 tests, 12 stderr failures, 0 stdout failures, 0 stderrB failures, 0 stdoutB failures, 0 post failures ==',
+     [125, 113, 12, 0, 0, 0, 0, 0, 0, 0]),
+    # zsh
+    ('46 successful test scripts, 0 failures, 1 skipped',
+     [47, 46, 0, 0, 1, 0, 0, 0, 0, 0]),
+    # glog
+    ('Passed 3 tests',
+     [3, 3, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # hdf5
+    ('Testing h5repack h5repack_szip.h5 -f dset_szip:GZIP=1                  -SKIP-',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('Verifying h5repack h5repack_szip.h5 -f dset_szip:GZIP=1                -SKIP-',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('Verifying h5dump output -f GZIP=1 -m 1024                             *FAILED*',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('Testing h5dump output -f GZIP=1 -m 1024                               *FAILED*',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('Testing h5repack --metadata_block_size=8192                            PASSED',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('Verifying h5diff output h5repack_layout.h5 out-meta_long.h5repack_layo PASSED',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # libconfig
+    ('3 tests; 3 passed, 0 failed',
+     [3, 3, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # libogg
+    ('testing page spill expansion... 0, (0),  granule:0 1, (1),  granule:4103 2, (2),  granule:5127 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing max packet segments... 0, (0),  granule:0 1, (1),  granule:261127 2, (2),  granule:262151 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing very large packets... 0, (0),  granule:0 1, (1),  granule:1031 2, (2), 3, (3),  granule:4103 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing continuation resync in very large packets... 0, 1, 2, (2), 3, (3),  granule:4103 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing zero data page (1 nil packet)... 0, (0),  granule:0 1, (1),  granule:1031 2, (2),  granule:2055 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Testing search for capture... ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Testing recapture... ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # libvorbis
+    ('    vorbis_1ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('    vorbis_2ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('    vorbis_7ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('    vorbis_8ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # pth
+    ('OK - ALL TESTS SUCCESSFULLY PASSED.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # casync (uses meson / ninja)
+    ("ninja: Entering directory `builddir'\n"
+     '[0/1] /usr/bin/python3 -u /usr/bin/meson test --no-rebuild --print-errorlogs\n'
+     ' 1/16 test-script.sh                          OK      21.20 s \n'
+     ' 2/16 test-script-sha256.sh                   OK      23.13 s \n'
+     ' 3/16 test-script-gzip.sh                     OK      20.91 s \n'
+     ' 4/16 test-script-xz.sh                       OK      29.97 s \n'
+     ' 5/16 test-nbd.sh                             OK       0.91 s \n'
+     ' 6/16 test-fuse.sh                            OK       1.25 s \n'
+     ' 7/16 test-cachunk                            OK       0.02 s \n'
+     ' 8/16 test-cachunker                          OK       0.70 s \n'
+     ' 9/16 test-cachunker-histogram                OK       2.04 s \n'
+     '10/16 test-cadigest                           OK      10.01 s \n'
+     '11/16 test-caencoder                          OK       0.05 s \n'
+     '12/16 test-camakebst                          OK       3.21 s \n'
+     '13/16 test-caorigin                           OK       0.00 s \n'
+     '14/16 test-casync                             OK       0.74 s \n'
+     '15/16 test-cautil                             OK       0.00 s \n'
+     '16/16 test-util                               OK       0.01 s \n'
+     '\n'
+     'OK:        16\n'
+     'FAIL:       0\n'
+     'SKIP:       0\n'
+     'TIMEOUT:    0\n',
+     [16, 16, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # gstreamer (uses meson / ninja)
+    ('+ meson test -C builddir\n'
+     "ninja: Entering directory `builddir'\n"
+     'ninja: no work to do.\n'
+     '1/6 gst_gst                                  OK       0.40 s\n'
+     '2/6 gst_gstabi                               FAIL     0.35 s\n'
+     '3/6 pipelines_stress                         OK       10.49 s\n'
+     '4/6 generic_sinks                            EXPECTEDFAIL 4.12 s\n'
+     '5/6 gst_gstcpp                               OK       0.37 s\n'
+     '6/6 libs_gstlibscpp                          OK       0.03 s\n'
+     'Ok:                   4\n'
+     'Expected Fail:        1\n'
+     'Fail:                 1\n'
+     'Unexpected Pass:      0\n'
+     'Skipped:              0\n'
+     'Timeout:              0\n',
+     [6, 4, 1, 1, 0, 0, 0, 0, 0, 0]),
+]
+
+backup_zero_test_data = count.zero_test_data
+
+def mock_zero_test_data():
+    pass
+
+
+class TestCount(unittest.TestCase):
+
+    def setUp(self):
+        count.zero_test_data()
+
+
+def test_generator(line, expected):
+    """
+    Generate a test for each line passed in
+    """
+    def test_parse_log(self):
+        """
+        test parse_log
+        expected = [total_tests,
+                    total_pass,
+                    total_fail,
+                    total_xfail,
+                    total_skip,
+                    counted_tests,
+                    counted_pass,
+                    counted_fail,
+                    counted_xfail,
+                    counted_skip]
+        """
+        content = '+ make check\n' + line
+        m_open = mock_open(read_data=content)
+        with patch('count.util.open_auto', m_open, create=True):
+            count.zero_test_data = mock_zero_test_data
+            count.parse_log('log')
+            count.zero_test_data = backup_zero_test_data
+
+        actual = [count.total_tests,
+                  count.total_pass,
+                  count.total_fail,
+                  count.total_xfail,
+                  count.total_skip,
+                  count.counted_tests,
+                  count.counted_pass,
+                  count.counted_fail,
+                  count.counted_xfail,
+                  count.counted_skip]
+
+        self.assertEqual(actual, expected)
+
+    return test_parse_log
+
+
+def test_setup():
+    for i, pat in enumerate(pats):
+        test_name = 'test_pat{}'.format(pat[0])
+        test = test_generator(pat[0], pat[1])
+        setattr(TestCount, test_name, test)
+
+
+# Run test_setup() to generate tests
+test_setup()
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=True)

tests/test_download.py

@@ -0,0 +1,164 @@
+from enum import Enum, auto
+import unittest
+from unittest.mock import patch, mock_open, call
+
+import pycurl
+
+import download
+
+
+class MockOpts(Enum):
+    URL = auto()
+    WRITEDATA = auto()
+    POSTFIELDS = auto()
+    FOLLOWLOCATION = auto()
+    FAILONERROR = auto()
+    CONNECTTIMEOUT = auto()
+    TIMEOUT = auto()
+    LOW_SPEED_LIMIT = auto()
+    LOW_SPEED_TIME = auto()
+
+
+def init_curl_instance(mock_curl):
+    instance = mock_curl.return_value
+    instance.URL = MockOpts.URL
+    instance.FOLLOWLOCATION = MockOpts.FOLLOWLOCATION
+    instance.FAILONERROR = MockOpts.FAILONERROR
+    instance.WRITEDATA = MockOpts.WRITEDATA
+    instance.POSTFIELDS = MockOpts.POSTFIELDS
+    return instance
+
+
+def test_opts(*opts):
+    if not opts:
+        raise Exception("no curl options specified")
+    if len(opts) != 2:
+        raise Exception("expected two args to setopt()")
+    key, val = opts
+    if key == MockOpts.WRITEDATA:
+        val.write(b'foobar')
+
+
+class TestDownload(unittest.TestCase):
+
+    @patch('download.pycurl.Curl')
+    def test_download_get_success_no_dest(self, test_curl):
+        """
+        Test successful GET request when dest is not set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo")
+        self.assertEqual(b'foobar', data.getvalue())
+
+    @patch('download.pycurl.Curl')
+    def test_download_set_basic(self, test_curl):
+        """
+        Test curl option settings set by default
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo")
+        calls = [
+            call().setopt(MockOpts.URL, 'foo'),
+            call().setopt(MockOpts.FOLLOWLOCATION, True),
+            call().setopt(MockOpts.FAILONERROR, True),
+        ]
+        test_curl.assert_has_calls(calls)
+
+    @patch('download.pycurl.Curl')
+    def test_download_set_post(self, test_curl):
+        """
+        Test setting of POSTFIELDS curl option
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo", post='postdata')
+        calls = [
+            call().setopt(MockOpts.POSTFIELDS, 'postdata'),
+        ]
+        test_curl.assert_has_calls(calls)
+
+    @patch('download.pycurl.Curl')
+    def test_download_get_failure_no_dest(self, test_curl):
+        """
+        Test failed GET request when dest is not set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        instance.perform.side_effect = pycurl.error
+        data = download.do_curl("foo")
+        self.assertIsNone(data)
+
+    @patch('download.sys.exit')
+    @patch('download.pycurl.Curl')
+    def test_download_get_failure_fatal(self, test_curl, test_exit):
+        """
+        Test failed GET request when is_fatal is set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        instance.perform.side_effect = pycurl.error
+        data = download.do_curl("foo", is_fatal=True)
+        test_exit.assert_called_once_with(1)
+
+    @patch('download.open', new_callable=mock_open)
+    @patch('download.pycurl.Curl')
+    def test_download_get_success_dest(self, test_curl, test_open):
+        """
+        Test successful GET request when dest is set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo", "testdest")
+        test_open.assert_called_once_with('testdest', 'wb')
+        test_open().write.assert_called_once_with(b'foobar')
+
+    @patch('download.os.path.exists')
+    @patch('download.open', new_callable=mock_open)
+    @patch('download.pycurl.Curl')
+    def test_download_get_write_fail_dest(self, test_curl, test_open, test_path):
+        """
+        Test failure to write to dest after successful GET request.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        test_open.side_effect = IOError
+        test_path.return_value = None
+        data = download.do_curl("foo", "testdest")
+        self.assertIsNone(data)
+
+    @patch('download.sys.exit')
+    @patch('download.os.path.exists')
+    @patch('download.open')
+    @patch('download.pycurl.Curl')
+    def test_download_write_fail_fatal(self, test_curl, test_open, test_path, test_exit):
+        """
+        Test fatal failure to write to dest after successful GET request.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        test_open.side_effect = IOError
+        test_path.return_value = None
+        data = download.do_curl("foo", "testdest", is_fatal=True)
+        test_exit.assert_called_once_with(1)
+
+    @patch('download.os.unlink')
+    @patch('download.os.path.exists')
+    @patch('download.open')
+    @patch('download.pycurl.Curl')
+    def test_download_write_fail_remove_dest(self, test_curl, test_open, test_path, test_unlink):
+        """
+        Test removal of dest following a write failure.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        test_open.side_effect = IOError
+        test_path.return_value = True
+        data = download.do_curl("foo", "testdest")
+        test_path.assert_called_once_with("testdest")
+        test_unlink.assert_called_once_with("testdest")
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_files.py

@@ -0,0 +1,533 @@
+import unittest
+import config
+import files
+import tempfile
+import os
+from unittest.mock import call, MagicMock
+import build
+from files import FileManager
+
+
+def mock_return(retval):
+    """
+    Simple mock method to set return value of a function
+    """
+    def mock_fn(_):
+        return retval
+
+    return mock_fn
+
+
+class TestFiles(unittest.TestCase):
+
+    def setUp(self):
+        conf = config.Config("")
+        pkg = build.Build()
+        self.fm = FileManager(conf, pkg)
+
+    def test_banned_path(self):
+        """
+        Test banned paths are detected
+        """
+        bad_paths = ["/etc/one",
+                     "/opt/two",
+                     "/V3/opt/two",
+                     "/V4/opt/two",
+                     "/VA/opt/two",
+                     "/usr/etc/three",
+                     "/usr/local/four",
+                     "/usr/src/five",
+                     "/var/six"]
+        for path in bad_paths:
+            self.assertTrue(FileManager.banned_path(path))
+
+    def test_push_package_file(self):
+        """
+        Test push_package_file with no package name specified (package name
+        should default to 'main'
+        """
+        self.assertFalse(self.fm.newfiles_printed)
+        self.fm.push_package_file('test-fn')
+        self.assertEqual(self.fm.packages['main'], set(['test-fn']))
+        self.assertTrue(self.fm.newfiles_printed)
+
+    def test_push_package_file_banned(self):
+        """
+        Test push_package_file with banned filename'
+        """
+        self.assertFalse(self.fm.newfiles_printed)
+        self.fm.push_package_file('/etc/test-fn')
+        self.fm.push_package_file('/V3/etc/test-fn')
+        self.fm.push_package_file('/V4/etc/test-fn')
+        self.fm.push_package_file('/VA/etc/test-fn')
+        self.assertTrue(self.fm.has_banned)
+        self.assertFalse(self.fm.newfiles_printed)
+
+    def test_push_package_file_dev(self):
+        """
+        Test push_package_file with dev package specified
+        """
+        self.fm.push_package_file('test-fn', 'dev')
+        self.assertEqual(self.fm.packages['dev'], set(['test-fn']))
+        self.assertTrue(self.fm.newfiles_printed)
+
+    def test_compat_exclude_keep_file(self):
+        """
+        Test compat_exclude with a file that shouldn't be excluded.
+        """
+        self.fm.config.config_opts['compat'] = True
+        self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so.1'))
+
+    def test_compat_exclude_exclude_file(self):
+        """
+        Test compat_exclude with a file that should be excluded.
+        """
+        self.fm.config.config_opts['compat'] = True
+        self.assertTrue(self.fm.compat_exclude('/usr/lib64/libfoo.so'))
+        self.assertTrue(self.fm.compat_exclude('/V3/usr/lib64/libfoo.so'))
+        self.assertTrue(self.fm.compat_exclude('/V4/usr/lib64/libfoo.so'))
+        self.assertTrue(self.fm.compat_exclude('/VA/usr/lib64/libfoo.so'))
+
+    def test_compat_exclude_not_compat_mode(self):
+        """
+        Test compat_exclude with a file that should be excluded but isn't
+        because the package isn't being run in compat mode.
+        """
+        self.fm.config.config_opts['compat'] = False
+        self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so'))
+        self.assertFalse(self.fm.compat_exclude('/V3/usr/lib64/libfoo.so'))
+        self.assertFalse(self.fm.compat_exclude('/V4/usr/lib64/libfoo.so'))
+        self.assertFalse(self.fm.compat_exclude('/VA/usr/lib64/libfoo.so'))
+
+    def test_file_pat_match(self):
+        """
+        Test file_pat_match with good match and no replacement or prefix
+        specified.
+        """
+        self.fm.push_package_file = MagicMock()
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/V3/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/V4/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/VA/test-fn', 'main')
+
+    def test_file_pat_match_exclude(self):
+        """
+        Test file_pat_match with good match and filename in excludes list.
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.excludes.append('/test-fn')
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main'))
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'^/test-fn', 'main'))
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'^/test-fn', 'main'))
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_not_called()
+
+    def test_file_pat_match_replacement(self):
+        """
+        Test file_pat_match with replacement provided
+        """
+        self.fm.push_package_file = MagicMock()
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/testfn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V3/testfn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V4/testfn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/VA/testfn', 'main')
+
+    def test_file_pat_match_replacement_no_glob(self):
+        """
+        Test file_pat_match with replacement provided but no glob set
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.config.config_opts['no_glob'] = True
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V3/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V4/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/VA/test-fn', 'main')
+
+    def test_file_windows_exe_not_allowed(self):
+        """
+        Test that Windows exe and dll files are excluded
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.config.config_opts['allow_exe'] = False
+        self.fm.push_file('/usr/bin/foo.exe', 'test')
+        self.fm.push_file('/usr/lib64/foo.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', 'test')
+        self.assertIn('/usr/bin/foo.exe', self.fm.excludes)
+        self.assertIn('/usr/lib64/foo.dll', self.fm.excludes)
+        self.assertIn('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', self.fm.excludes)
+        self.assertIn('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', self.fm.excludes)
+
+    def test_file_windows_exe_allowed(self):
+        """
+        Test that Windows exe and dll files are not excluded
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.config.config_opts['allow_exe'] = True
+        self.fm.push_file('/usr/bin/foo.exe', 'test')
+        self.fm.push_file('/usr/lib64/foo.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', 'test')
+        self.assertIn('/usr/bin/foo.exe', self.fm.files)
+        self.assertIn('/usr/lib64/foo.dll', self.fm.files)
+        self.assertIn('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', self.fm.files)
+        self.assertIn('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', self.fm.files)
+        self.assertNotIn('/usr/bin/foo.exe', self.fm.excludes)
+        self.assertNotIn('/usr/lib64/foo.dll', self.fm.excludes)
+        self.assertNotIn('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', self.fm.excludes)
+        self.assertNotIn('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', self.fm.excludes)
+
+    def test_file_pat_match_no_match(self):
+        """
+        Test file_pat_match with no match
+        """
+        self.fm.push_package_file = MagicMock()
+        self.assertFalse(self.fm.file_pat_match('test-fn', r'testfn', 'main'))
+        self.fm.push_package_file.assert_not_called()
+
+    def test_file_is_locale(self):
+        """
+        Test file_is_locale with locale filename not present in locale list
+        """
+        self.assertEqual(self.fm.locales, [])
+        self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo'))
+        self.assertEqual(self.fm.locales, ['loc'])
+
+    def test_file_is_locale_non_locale(self):
+        """
+        Test file_is_locale with non-locale filename
+        """
+        self.assertFalse(self.fm.file_is_locale('test-fn'))
+        self.assertEqual(self.fm.locales, [])
+
+    def test_file_is_locale_present(self):
+        """
+        Test file_is_locale with locale present in locale list
+        """
+        self.fm.locales.append('loc')
+        self.assertEqual(self.fm.locales, ['loc'])
+        self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo'))
+        self.assertEqual(self.fm.locales, ['loc'])
+
+    def test_push_file_autostart(self):
+        """
+        Test push_file to autostart package, this excludes the file.
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        autostart = '/usr/lib/systemd/system/some.target.wants/some'
+        self.fm.push_file(autostart, '')
+        calls = [call(autostart, 'autostart'), call('%exclude ' + autostart, 'services')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_file_custom_extras(self):
+        """
+        Test push_file to a custom extras package
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': ["/foobar"]}}
+        self.fm.push_file('/foobar', '')
+        calls = [call('/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/foobar', '')
+        calls = [call('/V3/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/foobar', '')
+        calls = [call('/V4/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/foobar', '')
+        calls = [call('/VA/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_empty(self):
+        """
+        Test push_file with glob extras empty match
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*rightglob']]}}
+        self.fm.push_file('/leftglobrightglob', '')
+        calls = [call('/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobrightglob', '')
+        calls = [call('/V3/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobrightglob', '')
+        calls = [call('/V4/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobrightglob', '')
+        calls = [call('/VA/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_left_match(self):
+        """
+        Test push_file with glob extras left content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*']]}}
+        self.fm.push_file('/leftglobrightglob', '')
+        calls = [call('/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobrightglob', '')
+        calls = [call('/V3/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobrightglob', '')
+        calls = [call('/V4/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobrightglob', '')
+        calls = [call('/VA/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_right_match(self):
+        """
+        Test push_file with glob extras right content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', '*rightglob']]}}
+        self.fm.push_file('/leftglobrightglob', '')
+        calls = [call('/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobrightglob', '')
+        calls = [call('/V3/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobrightglob', '')
+        calls = [call('/V4/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobrightglob', '')
+        calls = [call('/VA/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_leftright_match(self):
+        """
+        Test push_file with glob extras leftright content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*rightglob']]}}
+        self.fm.push_file('/leftglobstuffrightglob', '')
+        calls = [call('/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobstuffrightglob', '')
+        calls = [call('/V3/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobstuffrightglob', '')
+        calls = [call('/V4/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobstuffrightglob', '')
+        calls = [call('/VA/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_multi_match(self):
+        """
+        Test push_file with multiple globs extras content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*', '*rightglob']]}}
+        self.fm.push_file('/leftglobstuff/stuffrightglob', '')
+        calls = [call('/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobstuff/stuffrightglob', '')
+        calls = [call('/V3/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobstuff/stuffrightglob', '')
+        calls = [call('/V4/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobstuff/stuffrightglob', '')
+        calls = [call('/VA/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_file_setuid(self):
+        """
+        Test push_file with fname in setuid list
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.setuid.append('test')
+        self.fm.push_file('test', '')
+        calls = [call('%attr(4755, root, root) test', 'setuid')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+
+    def test_push_file_match(self):
+        """
+        Test push_file with match in pattern list
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.push_file('/usr/bin/test', '')
+        self.fm.push_package_file.assert_called_once_with('/usr/bin/test', 'bin')
+        self.fm.push_file('/V3/usr/bin/test', '')
+        self.fm.push_package_file.assert_has_calls([call('/V3/usr/bin/test', 'bin')])
+        self.fm.push_file('/V4/usr/bin/test', '')
+        self.fm.push_package_file.assert_has_calls([call('/V4/usr/bin/test', 'bin')])
+        self.fm.push_file('/VA/usr/bin/test', '')
+        self.fm.push_package_file.assert_has_calls([call('/VA/usr/bin/test', 'bin')])
+
+    def test_push_file_match_pkg_name_dependency(self):
+        """
+        Test push_file with match in the list on the single item that is
+        dependent on the pkg_name.
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.push_file('/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_called_once_with('/usr/share/doc/testball/*', 'doc')
+        self.fm.push_file('/V3/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_has_calls([call('/V3/usr/share/doc/testball/*', 'doc')])
+        self.fm.push_file('/V4/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_has_calls([call('/V4/usr/share/doc/testball/*', 'doc')])
+        self.fm.push_file('/VA/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_has_calls([call('/VA/usr/share/doc/testball/*', 'doc')])
+
+    def test_push_file_no_match(self):
+        """
+        Test push_file with no pattern match on the file name. Should just push
+        the unmodified filename once.
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.push_file('doesntmatcha thing', '')
+        self.fm.push_package_file.assert_called_once_with('doesntmatcha thing')
+
+    def test_remove_file(self):
+        """
+        Test remove_file with filename in files list and main package
+        """
+        self.fm.files.add('test')
+        self.fm.packages['main'] = ['test']
+        self.assertIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.files_blacklist)
+        self.assertIn('test', self.fm.packages['main'])
+        self.fm.remove_file('test')
+        self.assertNotIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.packages['main'])
+        self.assertIn('test', self.fm.files_blacklist)
+
+    def test_remove_file_not_present(self):
+        """
+        Test remove_file with filename not in files list.
+        """
+        self.assertNotIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.files_blacklist)
+        self.fm.remove_file('test')
+        self.assertNotIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.files_blacklist)
+
+    def test_clean_directories(self):
+        """
+        Test clean_directories with a directory in the list
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("/directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"], set(["/file1", "/file2"]))
+
+
+    def test_clean_directories_with_dir(self):
+        """
+        Test clean_directories with a %dir directory in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%dir /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%dir /directory", "/file1", "/file2"]))
+
+
+    def test_clean_directories_with_symlink_to_dir(self):
+        """
+        Test clean_directories with a symlink to a directory in the list. The
+        symlink should remain, but the directory should be cleaned.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            dirname = os.path.join(tmpd, "directory")
+            linkname = os.path.join(tmpd, "symlink")
+            os.mkdir(dirname)
+            os.symlink(dirname, linkname)
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("/directory")
+            self.fm.packages["main"].add("/symlink")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["/symlink"]))
+
+
+    def test_clean_directories_with_symlink_to_explicit_dir(self):
+        """
+        Test clean_directories with a symlink to a %dir directory in the list.
+        The symlink and directory should both remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            dirname = os.path.join(tmpd, "directory")
+            linkname = os.path.join(tmpd, "symlink")
+            os.mkdir(dirname)
+            os.symlink(dirname, linkname)
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%dir /directory")
+            self.fm.packages["main"].add("/symlink")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%dir /directory", "/symlink"]))
+
+
+    def test_clean_directories_with_doc(self):
+        """
+        Test clean_directories with a %doc directive in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%doc /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%doc /directory", "/file1", "/file2"]))
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_general.py

@@ -0,0 +1,24 @@
+import subprocess
+import unittest
+
+
+class TestGeneral(unittest.TestCase):
+
+    def test_ConfigParser_regressions(self):
+        """
+        Make sure ConfigParser is always called with the required
+        interpolation=None argument
+        """
+        grep_cmd = ["grep", "-re",
+                    "ConfigParser(.*(^interpolation=None).*)",
+                    "autospec"]
+        try:
+            output = subprocess.check_output(grep_cmd).decode('utf-8')
+        except subprocess.CalledProcessError as e:
+            output = e.output.decode('utf-8')
+
+        self.assertEqual(output.strip(), "")
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=True)

tests/test_license.py

@@ -0,0 +1,320 @@
+from contextlib import redirect_stdout
+from io import BytesIO, StringIO
+import os
+import tempfile
+import unittest
+from unittest.mock import patch, mock_open, MagicMock
+
+import pycurl
+
+import config
+import download
+import license
+import util
+
+
+class TestLicense(unittest.TestCase):
+
+    def setUp(self):
+        license.licenses = []
+
+    def test_add_license(self):
+        """
+        Test add_license from valid string, Apache-2 should be translated to
+        Apache-2.0
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        self.assertTrue(license.add_license('Apache-2', conf.license_translations, conf.license_blacklist))
+        self.assertIn('Apache-2.0', license.licenses)
+
+    def test_add_license_present(self):
+        """
+        Test add_license from valid string, but license is already present in
+        the licenses list. Should return True and should not modify the
+        licenses list. GPL-3 translates to GPL-3.0.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        license.licenses.append('GPL-3.0')
+        self.assertTrue(license.add_license('GPL-3', conf.license_translations, conf.license_blacklist))
+        self.assertEqual(['GPL-3.0'], license.licenses)
+
+    def test_add_license_blacklisted(self):
+        """
+        Test add_license from string in license_blacklist. Should return False
+        and should not modify the licenses list.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        # sanity check to make sure the licenses list is empty before the later
+        # assertIn() call
+        self.assertEqual(license.licenses, [])
+
+        self.assertFalse(license.add_license('License', conf.license_translations, conf.license_blacklist))
+        self.assertNotIn('License', license.licenses)
+
+    def test_license_from_copying_hash(self):
+        """
+        Test license_from_copying_hash with valid license file
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+        self.assertIn('GPL-3.0', license.licenses)
+
+    def test_license_from_copying_hash_no_license_show(self):
+        """
+        Test license_from_copying_hash with invalid hash and no license_show
+        set
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        # remove the hash from license_hashes
+        del(conf.license_hashes[license.util.get_sha1sum('tests/COPYING_TEST')])
+        conf.license_show = "license.show.url"
+        license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+
+        self.assertEqual(license.licenses, [])
+
+    def test_license_from_copying_hash_bad_license(self):
+        """
+        Test license_from_copying_hash with invalid license file
+        """
+        conf = config.Config("")
+        content = util.get_contents("tests/COPYING_TEST").replace(b"GNU", b"SNU")
+        m_open = MagicMock()
+        m_open.__str__.return_value = content
+
+        with patch('license.get_contents', m_open, create=True):
+            license.license_from_copying_hash('copying.txt', '', conf, '')
+
+        self.assertEqual(license.licenses, [])
+
+    def test_license_from_copying_hash_license_server_excep(self):
+        """
+        Test license_from_copying_hash with license server when pycurl raises
+        an exception.
+        """
+        class MockCurl():
+            URL = None
+            WRITEDATA = None
+            POSTFIELDS = None
+            FOLLOWLOCATION = 0
+            FAILONERROR = False
+            CONNECTTIMEOUT = 0
+            TIMEOUT = 0
+            LOW_SPEED_LIMIT = 0
+            LOW_SPEED_TIME = 0
+            def setopt(_, __, ___):
+                pass
+
+            def perform(_):
+                raise pycurl.error('Test Exception')
+
+            def close(_):
+                pass
+
+        # set the mock curl
+        download.pycurl.Curl = MockCurl
+
+        conf = config.Config("")
+        conf.license_fetch = 'license.server.url'
+
+        # let's check that the proper thing is being printed as well
+        out = StringIO()
+        with redirect_stdout(out):
+            with self.assertRaises(SystemExit):
+                license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+
+        self.assertIn('Unable to fetch license.server.url: Test Exception', out.getvalue())
+
+        # unset the manual mock
+        download.pycurl.Curl = pycurl.Curl
+
+    def test_license_from_copying_hash_license_server(self):
+        """
+        Test license_from_copying_hash with license server. This is heavily
+        mocked.
+        """
+        class MockBytesIO(BytesIO):
+            """
+            Mock class for BytesIO to set returnvalue of BytesIO.getvalue()
+            """
+            def getvalue(_):
+                return 'GPL-3.0'.encode('utf-8')
+
+        # set the mocks
+        download.BytesIO = MockBytesIO
+
+        class MockCurl():
+            URL = None
+            WRITEDATA = None
+            POSTFIELDS = None
+            FOLLOWLOCATION = 0
+            FAILONERROR = False
+            CONNECTTIMEOUT = 0
+            TIMEOUT = 0
+            LOW_SPEED_LIMIT = 0
+            LOW_SPEED_TIME = 0
+            def setopt(_, __, ___):
+                pass
+
+            def perform(_):
+                pass
+
+            def close(_):
+                pass
+
+            def getinfo(_, __):
+                return 200
+
+        # set the mock curl
+        download.pycurl.Curl = MockCurl
+
+        conf = config.Config("")
+        conf.license_fetch = 'license.server.url'
+
+        # let's check that the proper thing is being printed as well
+        out = StringIO()
+        with redirect_stdout(out):
+            license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+
+        self.assertIn('GPL-3.0', license.licenses)
+        self.assertIn('License     :  GPL-3.0  (server)', out.getvalue())
+
+        # unset the manual mock
+        download.BytesIO = BytesIO
+
+        # unset the manual mock
+        download.pycurl.Curl = pycurl.Curl
+
+    def test_scan_for_licenses(self):
+        """
+        Test scan_for_licenses in temporary directory with valid license file
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        with open('tests/COPYING_TEST', 'rb') as copyingf:
+            content = copyingf.read()
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            # create the copying file
+            with open(os.path.join(tmpd, 'COPYING'), 'w') as newcopyingf:
+                newcopyingf.write(content.decode('utf-8'))
+            # create some cruft for testing
+            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
+                with open(os.path.join(tmpd, testf), 'w') as newtestf:
+                    newtestf.write('test content')
+            license.scan_for_licenses(tmpd, conf, '')
+
+        self.assertIn('GPL-3.0', license.licenses)
+
+    def test_scan_for_licenses_none(self):
+        """
+        Test scan_for_licenses in temporary directory with no matching files.
+        Should not add any licenses, should print a fatal message, should exit
+        with a status code of 1.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        with tempfile.TemporaryDirectory() as tmpd:
+            # create some cruft for testing
+            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
+                with open(os.path.join(tmpd, testf), 'w') as newtestf:
+                    newtestf.write('test content')
+            # let's check that the proper thing is being printed as well
+            out = StringIO()
+            with redirect_stdout(out):
+                with self.assertRaises(SystemExit) as thread:
+                    license.scan_for_licenses(tmpd, conf, '')
+
+        self.assertEqual(thread.exception.code, 1)
+        self.assertIn("Cannot find any license", out.getvalue())
+        self.assertEqual(license.licenses, [])
+
+    def test_scan_for_licenses_skip(self):
+        """
+        Test scan_for_licenses in temporary directory with licenses to skip
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        conf.license_skips = [['COPYING']]
+        with open('tests/COPYING_TEST', 'rb') as copyingf:
+            content = copyingf.read()
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            # create the copying file
+            with open(os.path.join(tmpd, 'COPYING'), 'w') as newcopyingf:
+                newcopyingf.write(content.decode('utf-8'))
+            # create some cruft for testing
+            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
+                with open(os.path.join(tmpd, testf), 'w') as newtestf:
+                    newtestf.write('test content')
+            # let's check that the proper thing is being printed as well
+            out = StringIO()
+            with redirect_stdout(out):
+                with self.assertRaises(SystemExit) as thread:
+                    license.scan_for_licenses(tmpd, conf, '')
+
+        self.assertEqual(thread.exception.code, 1)
+        self.assertIn("Cannot find any license", out.getvalue())
+        self.assertEqual(license.licenses, [])
+
+    def test_scan_for_licenses_skip_prefix_slash(self):
+        """
+        Test scan_for_licenses in temporary directory with licenses to skip
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        conf.license_skips = [['', 'COPYING']]
+        with open('tests/COPYING_TEST', 'rb') as copyingf:
+            content = copyingf.read()
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            # create the copying file
+            with open(os.path.join(tmpd, 'COPYING'), 'w') as newcopyingf:
+                newcopyingf.write(content.decode('utf-8'))
+            # create some cruft for testing
+            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
+                with open(os.path.join(tmpd, testf), 'w') as newtestf:
+                    newtestf.write('test content')
+            # let's check that the proper thing is being printed as well
+            out = StringIO()
+            with redirect_stdout(out):
+                with self.assertRaises(SystemExit) as thread:
+                    license.scan_for_licenses(tmpd, conf, '')
+
+        self.assertEqual(thread.exception.code, 1)
+        self.assertIn("Cannot find any license", out.getvalue())
+        self.assertEqual(license.licenses, [])
+
+    def test_load_specfile(self):
+        """
+        Test load_specfile with populated license list. This method is not
+        normally tested but there is some logic here.
+        """
+        class MockSpecfile(object):
+            licenses = []
+
+        license.licenses = ['GPL-3.0', 'MIT']
+        specfile = MockSpecfile()
+        license.load_specfile(specfile)
+        self.assertEqual(specfile.licenses, license.licenses)
+
+    def test_load_specfile_none(self):
+        """
+        Test load_specfile with unpopulated license list. This method is not
+        normally tested but there is some logic here.
+        """
+        class MockSpecfile(object):
+            licenses = []
+
+        license.licenses = []
+        specfile = MockSpecfile()
+        license.load_specfile(specfile)
+        self.assertEqual(specfile.licenses, [license.default_license])
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_pkg_integrity.py

@@ -0,0 +1,450 @@
+from io import BytesIO
+import json
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import config
+import download
+import pkg_integrity
+
+
+TESTDIR = os.path.join(os.getcwd(), "tests/testfiles/pkg_integrity")
+TESTKEYDIR = os.path.join(TESTDIR, "testkeys")
+
+PACKAGE_URL = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz"
+NO_SIGN_PKT_URL = "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz"
+NOSIGN_PKT_URL_BAD = "http://gnu.mirrors.pair.com/savannah/savannah/quagga/bad_quagga-1.1.0.tar.gz"
+NOSIGN_PKT_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz"
+NOSIGN_SIGN_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz.asc"
+GNOME_SHA256_PKG = "https://download.gnome.org/sources/pygobject/3.24/pygobject-3.24.0.tar.xz"
+QT_SHA256_PKG = "https://download.qt.io/official_releases/qt/5.12/5.12.4/submodules/qtspeech-everywhere-src-5.12.4.tar.xz"
+KEYID = "EC2392F2EDE74488680DA3CF5F2B4756ED873D23"
+
+
+def mock_download_do_curl(url, dst=None):
+    bad_sigs = ["http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.asc",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.sign"]
+    if not dst:
+        return BytesIO(b'foobar')
+    src = os.path.join(TESTDIR, os.path.basename(url))
+    if dst and os.path.isfile(src):
+        shutil.copyfile(src, dst)
+        return dst
+    else:
+        return None
+    if url in bad_sigs:
+        return None
+
+
+@patch('download.do_curl', mock_download_do_curl)
+class TestCheckFn(unittest.TestCase):
+
+    def test_check_matching_sign_url(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            pkey = "023A4420C7EC6914.pkey"
+            shutil.copy(os.path.join(TESTKEYDIR, pkey), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd)
+            result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertTrue(result)
+
+    def test_check_with_existing_sign(self):
+        """ Download signature for local verification """
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd)
+            result = pkg_integrity.check(NOSIGN_PKT_URL, conf)
+            self.assertTrue(result)
+
+
+@patch('download.do_curl', mock_download_do_curl)
+class TestDomainBasedVerifiers(unittest.TestCase):
+
+    def run_test_for_domain(self, Verifier, url):
+        with tempfile.TemporaryDirectory() as tmpd:
+            filen = os.path.basename(url)
+            shutil.copy(os.path.join(TESTDIR, filen), tmpd)
+            package_path = os.path.join(tmpd, filen)
+            verifier = Verifier(**{'package_path': package_path,
+                                   'url': url})
+            return verifier.verify()
+        return None
+
+    def _mock_fetch_shasum(url):
+        return (
+                "100395496483fcea7ba03fc1655c7a770f7f2e12e93be8bda2e31fec42debde0 pygobject-3.24.0.news\n"
+                "ae417db3be2a197b403bba6472cfb35a6e642cd802660832acb9c96123f79463  pygobject-3.24.0.changes\n"
+                "4e228b1c0f36e810acd971fad1c7030014900d8427c308d63a560f3f1037fa3c pygobject-3.24.0.tar.xz"
+                )
+
+    @patch('pkg_integrity.GnomeOrgVerifier.fetch_shasum', _mock_fetch_shasum)
+    def test_gnome_org(self):
+        result = self.run_test_for_domain(pkg_integrity.GnomeOrgVerifier, GNOME_SHA256_PKG)
+        self.assertTrue(result)
+
+    @patch('pkg_integrity.QtIoVerifier.fetch_shasum')
+    def test_qt_io(self, test_fetch):
+        test_fetch.return_value = "2ff9660fb3f5663c9161f491d1a304db62691720136ae22c145ef6a1c94b90ec  qtspeech-everywhere-src-5.12.4.tar.xz\n"
+        result = self.run_test_for_domain(pkg_integrity.QtIoVerifier, QT_SHA256_PKG)
+        self.assertTrue(result)
+
+
+@patch('download.do_curl', mock_download_do_curl)
+class TestGPGVerifier(unittest.TestCase):
+
+    def test_from_url(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertTrue(result)
+
+    def test_invalid_key(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd)
+            with open(os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD) + ".asc"), 'w') as ofile:
+                ofile.write("Invalid signature")
+            result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, conf)
+            self.assertIsNone(result)
+
+    def test_key_not_found(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd)
+            result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, conf)
+            self.assertIsNone(result)
+
+    def test_from_disk(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
+            out_key = out_file + ".asc"
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd)
+            result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key, conf)
+            self.assertTrue(result)
+
+    def test_non_matchingsig(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
+            f = open(out_file, 'wb')
+            f.write(b'made up date that will fail check')
+            f.close()
+            with self.assertRaises(SystemExit) as a:
+                pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertEqual(a.exception.code, 1)
+
+    def test_result_on_non_existent_pkg_path(self):
+        conf = config.Config('')
+        conf.rewrite_config_opts = unittest.mock.Mock()
+        conf.config_opts['verify_required'] = False
+        result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz',
+                                         'NonExistentPKG.tar.gz',
+                                         'NonExistentKey.asc',
+                                         conf)
+        self.assertIsNone(result)
+
+    def test_result_on_nosign_package(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd)
+            result = pkg_integrity.check(NO_SIGN_PKT_URL, conf)
+            self.assertIsNone(result)
+
+    @patch.object(pkg_integrity.GPGCli, 'exec_cmd')
+    @patch('pkg_integrity.parse_gpg_packets')
+    def test_result_multiple_sig(self, mock_parse, mock_exec):
+        """Test verification of first signature of tarball with multiple signatures."""
+        def packets_separator(filename, **kwargs):
+            if filename.endswith('.pkey'):
+                packets = [
+                    {
+                        'offset': 528,
+                        'length': 37,
+                        'type': 'user ID',
+                        'email': 'user1@example.com',
+                    },
+                ]
+            elif filename.endswith('.asc'):
+                packets = [
+                    {
+                        'offset': 0,
+                        'length': 543,
+                        'type': 'signature',
+                        'keyid': '023A4420C7EC6914',
+                    },
+                    {
+                        'offset': 543,
+                        'length': 543,
+                        'type': 'signature',
+                        'keyid': '12345678DEADCAFE',
+                    },
+                ]
+            return packets
+        mock_parse.side_effect = packets_separator
+
+        mock_exec.return_value = (b'', b'', 0)
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertTrue(result)
+            self.assertEqual(mock_parse.call_count, 4)
+            self.assertEqual(mock_exec.call_count, 3)
+            self.assertEqual(pkg_integrity.EMAIL, "user1@example.com")
+            self.assertEqual(pkg_integrity.KEYID, "023A4420C7EC6914")
+
+    @patch.object(pkg_integrity.GPGCli, 'exec_cmd')
+    @patch('pkg_integrity.parse_gpg_packets')
+    def test_result_multiple_sig_no_separators(self, mock_parse, mock_exec):
+        """Test skipping of sig verification in the multiple sig case when packet separators are absent."""
+        def packets_no_separator(filename, **kwargs):
+            if filename.endswith('.pkey'):
+                packets = [
+                    {
+                        'type': 'user ID',
+                        'email': 'user2@example.com',
+                    },
+                ]
+            elif filename.endswith('.asc'):
+                packets = [
+                    {
+                        'type': 'signature',
+                        'keyid': '023A4420C7EC6914',
+                    },
+                    {
+                        'type': 'signature',
+                        'keyid': 'DEADCAFEC7EC6914',
+                    },
+                ]
+            return packets
+        mock_parse.side_effect = packets_no_separator
+
+        mock_exec.return_value = (b'', b'', 0)
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            with self.assertRaises(SystemExit) as msg:
+                result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertEqual(msg.exception.code, 1)
+            self.assertEqual(mock_parse.call_count, 4)
+            self.assertEqual(mock_exec.call_count, 2)
+            self.assertEqual(pkg_integrity.EMAIL, "user2@example.com")
+            self.assertEqual(pkg_integrity.KEYID, "023A4420C7EC6914")
+
+
+class TestInputGetter(unittest.TestCase):
+
+    def test_timput(self):
+        ig = pkg_integrity.InputGetter(default='N', timeout=2)
+        answer = ig.get_answer()
+        self.assertIsNone(answer)
+        ig = pkg_integrity.InputGetter(default='Y', timeout=2)
+        answer = ig.get_answer()
+        self.assertIsNone(answer)
+
+
+class TestUtils(unittest.TestCase):
+
+    def test_get_verifier(self):
+        x = pkg_integrity.get_verifier('file.abcd')
+        self.assertEqual(x, None)
+
+        y = pkg_integrity.get_verifier('xorriso-1.4.6.tar.gz')(
+                package_path='',
+                url='http://ftp.gnu.org/gnu/xorriso/xorriso-1.4.6.tar.gz',
+                package_check='http://ftp.gnu.org/gnu/xorriso/xorriso-1.4.6.tar.gz.asc'
+                )
+        self.assertTrue(isinstance(y, pkg_integrity.GPGVerifier))
+
+    def test_parse_gpg_packets_for_keyid(self):
+        """Test parse_gpg_packets() to retrieve keyid info."""
+        def check_packets(algo, key_id, packet_count, packet_with_val):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                packets = pkg_integrity.parse_gpg_packets(tmpf.name)
+                self.assertIsNotNone(packets)
+                self.assertEqual(len(packets), packet_count)
+                self.assertEqual(packets[packet_with_val]["keyid"], key_id)
+                tmpf.close()
+
+        check_packets(KEY_ALGO17, '8AFAFCD242818A52', 6, 1)
+        check_packets(KEY_ALGO1, '330239C1C4DAFEE1', 1, 0)
+
+    def test_get_keyid(self):
+        """Test get_keyid() to retrieve key ID from GPG key or signature."""
+        def check_get_keyid(algo, key_id):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                result = pkg_integrity.get_keyid(tmpf.name)
+                self.assertEqual(result, key_id)
+                tmpf.close()
+
+        check_get_keyid(KEY_ALGO17, '8AFAFCD242818A52')
+        check_get_keyid(KEY_ALGO1, '330239C1C4DAFEE1')
+
+    def test_get_keyid_none(self):
+        """Test get_keyid() when the key name is invalid."""
+        false_name = '/false/name'
+        self.assertTrue(pkg_integrity.get_keyid(false_name) is None)
+
+    def _mock_download_file(url, dst=None):
+        # make return codes match by url to ensure we are using the expected signature type
+        if url in ("http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz.sig",
+                   "http://download.savannah.gnu.org/releases/quilt/quilt-0.65.tar.gz.asc",
+                   "http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2.sign"):
+            return os.path.join(dst, os.path.basename(url))
+        return None
+
+    @patch('download.do_curl', _mock_download_file)
+    def test_get_signature_url(self):
+        url_from_gnu = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz"
+        url_from_gnu1 = "http://download.savannah.gnu.org/releases/quilt/quilt-0.65.tar.gz"
+        url_from_gnu2 = "http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2"
+
+        self.assertEqual(pkg_integrity.get_signature_file(url_from_gnu, '.')[-4:], '.sig')
+        self.assertEqual(pkg_integrity.get_signature_file(url_from_gnu1, '.')[-4:], '.asc')
+        self.assertEqual(pkg_integrity.get_signature_file(url_from_gnu2, '.')[-5:], '.sign')
+
+    def test_parse_gpg_packets_for_email(self):
+        """Test parse_gpg_packets() to retrieve email info."""
+        def check_packets(algo, email, packet_count, packet_with_val):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                packets = pkg_integrity.parse_gpg_packets(tmpf.name)
+                self.assertIsNotNone(packets)
+                self.assertEqual(len(packets), packet_count)
+                if packet_with_val:
+                    self.assertEqual(packets[packet_with_val]["email"], email)
+                tmpf.close()
+
+        check_packets(KEY_ALGO17, 'kislyuk@gmail.com', 6, 0)
+        check_packets(KEY_ALGO1, None, 1, None)
+
+    def test_get_email(self):
+        """Test get_email() to retrieve email info from GPG key."""
+        def check_get_email(algo, email):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                result = pkg_integrity.get_email(tmpf.name)
+                self.assertEqual(result, email)
+                tmpf.close()
+
+        check_get_email(KEY_ALGO17, 'kislyuk@gmail.com')
+        check_get_email(KEY_ALGO1, None)
+
+KEY_ALGO1 = b"""\
+-----BEGIN PGP SIGNATURE-----
+
+iQEcBAABAgAGBQJX1yrCAAoJEDMCOcHE2v7hwhcH/AqhJ/vqkCOo09Yh89bYRFyb
+O5yEpaRV7vo4qoqXht3fQiR3KC3lSKybtScihbQ0xcTeBzSwGPMKPRpOqXEKRnwY
+9Zq8ev72Ixi5yVsKdSKjoeM4smXJdQolKnrKy0chsOMzu7cxk7hwejplIMjycKza
+g3HM6jtw6v10JDj6a5SJPkrufj2eIHq1enn2di9q9+yyAJGiuWi3ABfINmL3Y9sa
+zODoMBD/B7LCZ1Zv6dG7kaN1XS82crOmXrtgcWslKxfGPkdW9SGQdhCtm6f/Z5/w
+29adzXFicobZBFBHVnbQ0iRf39omkNxPOhMYwLVQFprzjOPm/DDHSztZj87jbOM=
+=Qe1y
+-----END PGP SIGNATURE-----
+"""
+
+KEY_ALGO17 = b"""\
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQGiBEYdIyURBACrG5G9L7R2uLGnCer+liVhsn5vwJIwIpLtVTb7Z3AcOoumza66
+5JBZtY9RSsNRbMcuqxPWeXnl8jhs513O0dZihTL+/cLMD1RJAXRlf0eDxAYl2TiD
+7AdH+y6HGHljn2IkH5jIzdTgNXdJZ8BArkFZP7an+ZLfE5RgSQ4QqZpuxwCgiq9+
+fjVoHSvGJHlcN4SiBBn8cCkEAKsYT7rp7N2GaA6b4XjLr2bVnv3MV431EhxTHynX
+mcnpT91ItMAkjPWcMSWWBhYJ/rgi2KYh5oAMhcLt2gCU58TxMsxA2rPEocHFDzKY
+Yj9HOXCa4XasL4R+Jyu4/Flh7Fqvrts5iZViDx/hPo1O5HhM7VuVnDxOc6yVPDB1
+OGj5A/9lCCn2awr4XxEWsYnBDtoqzfb8jwIDfS4xtqCAaegiXr4P1GGgkDtlW0XC
+5k2giFiqD0l928y7YV7Mumw2hjqECyPwDDbsEMfJasWEnAG7hBmAfHtIwoC8563d
+3QM16Wu33xXB+rbss/mH95oUOEtkR1m9HCAOZ+R/T9WVdwTKN7QiQW5kcmV5IEtp
+c2x5dWsgPGtpc2x5dWtAZ21haWwuY29tPohoBBMRAgAoBQJXdAKuAhsDBQkSzAMA
+BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCK+vzSQoGKUgbWAJsFQ1PeMKGA
+JC3GyQqoKIN65KibwQCfVWLmpSuP9Zmb9prjPDQyt8B0NXG0MkFuZHJleSBLaXNs
+eXVrICh3ZWF2ZXIpIDxraXNseXVrQG9jZi5iZXJrZWxleS5lZHU+iGYEExECACYF
+AkYdIyUCGwMFCRLMAwAGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCK+vzSQoGK
+UiE9AKCADVzPhdseTQbrsVfZtqgIDqC/ZgCeL3UK2vHkLHBWiYzf1mCXzfgRpiG5
+Ag0ERh0jKxAIAImRscnCUdzp52n8KLPsyD/U+hn5YAUfGef0n6+eUuEh1ygH0J+H
+8W72K16dReY9X05Htod699TpHCWhmAL9DhqbAbjRtJ5B7hoTq6IS/FWbKRnsW/x7
+il/pY6s7P74J1y1NYTbDC+5spw3QCACieU8QdK54sOhUVgTWYC55mu1ycVLSYFjp
+me7Q6nKpcEq67zHx0QLmjFEryarlL7wEBvB33KzeAQ4thif8SHn3UM5TPkGlMs0h
+T/BFNYH+dMd703HS+rwv1/vs87PxhIb+QqoK9BwVbZdLEd12D1Amk/oX/hZgELUO
+tBgMj5Zbk+Jl6AWp8RgRWnzV1JSu2Zj/1wsAAwYH/1DwGUhpu9Sg3DgeEqp4tyyB
+qVUtTXNc6F3XVzeD3i7jahjdJAlMhi+rQrQNZXNVTMdHn7mec5Oi6urGR79eabFv
+cENWDHKj6QkMo4PCIi+y7TSoLyywQLxxxsDz5FGC5eTwSzo9Cdm6nwWhmCwNR2XG
+bb68/TVinfYCXyo+q6EfmOKDeBQxCTdiUkWnkMPPfw4VV2/7KPxuJWcJ+eENFlY1
+mHONQdJneIGpnXnvIT4peBPIVH4Q8FmZOzxEBvZQPKXvE3rxZYpFQBUmTb5kCeBb
+VDRHm4iUY2IR1VFk2ExCCnYS6ds4Alt5sy4jQGY9ttOAXDX9hUBUpomG6AAvafqI
+TwQYEQIADwUCRh0jKwIbDAUJEswDAAAKCRCK+vzSQoGKUmKLAJ90lCzOl94ztnyT
+dImfPudUs7wYFQCeOS0T+YvYajEA99RWUnx2IH1rFKK5Ay4EV3QCMxEIAME79anw
++LZGB/d240txYXggyLefLskCJaIZrRoqxflTJCu6Gs6tTR8J29HO+GCuGKMMpLeF
+x9uSMU2VeO+3HlpW9MD4EPGPL1ZZGecftXwFTMLCcoiPkLGqMzidENgQpNTyKJpE
+vX1F3gIU5O53F35xeCWYSwNNAY9V+knNXG9syN59hrwa3Avr0Pu39lAABPKqKpqJ
+LLMtxPSDfJljaEpbeTr0SS18cGO5ytK+sH7jBhp5oBi79tcqV3zpDCh1crSfZewb
+Ugldem19fx3bRP7okYxSSJrDGjgte9jQWejeHRrUlSI9W8DEmVMo43CiEN6IG4aF
+fYLs1MQK25nzc58BAKK5PWll/moC9Ne3sF2Deuw91gLNLEbwlQto4M49IMCRB/9h
+9Ylaq+C4M3db46v/imwlSJmL7zVrbdENAY4aJEgpnSFNsgcFgnU0od+XTKiIVum+
+Fxr6PbB2DcAoMqeGsu6R+uAWFJiAMyqLrAkcMeYg3p6DxY17oCLcIiEm9hnlA0K8
+mYOw7Ykkb+oJqjhBUm5PFQH5KmXp/fbBuRiYrDfI/hBMw7J6XouH3vvrnIR5gBJM
+UvFkFt9RDMFQ2g+PjHbvZx1RMNMRJdw2uqAl+HHkF2a/Piyf+U6lBnKnV161FPbS
+mGKgbguN6y5SDfuDYHBH+GOmuYYDiU+BQnPhp0+fmEb2xZS4EzM7vcoCUzOAIbdl
+EV4v/tduwyLcWX+J3sIfCACxxmKDR1aKhdG6BHY2UmYIsp54xo/MOJmco2DlQUpj
+c45J8/4XYspXt5rI8fby81vHD2qMiDJ4aEYjDkmW+6XhuYjyr4xTqkvS9gMDggqH
+EccGqz7IoWrQPWewDGxVINq9rLU0l6puKR6aXwx8J9RwRYy0oECjSdETuP+NdxXd
+OVvnvBYMKSWWbIam1sjIiduczyVVRv8t1M5EKMG+KR/VS+hyjNZYduSp7OmettWT
+wyIG3LXiBHffFHPZ0kn3P/sZ8inBEzsjfiVm9mLL3yIL2C1YSY2uaiigPBPeNJM3
+QSnY6rSpfyGVKTC4/O9jX/+G0DJ7k0fdb7JwIS6Kd0huiK8EGBECAA8FAld0AjMC
+GwIFCRLMAwAAagkQivr80kKBilJfIAQZEQgABgUCV3QCMwAKCRD51uveZ3aeBPHd
+AP0WPpdXXqw2/kjuxydswKsI7I07uZlWeyou9zpath+0YgD8CW3o79TDafTHnpbo
+fX3HT+yO2BftAx+rzCymzllQLe+kcwCfeiLfkBj+ZyX9fhNT5YyiagmY5LoAn1TH
+qA34waak3JSxv16lZC0TqIuD
+=cHO/
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_specdescription.py

@@ -0,0 +1,252 @@
+import unittest
+from unittest.mock import mock_open, patch
+import config
+import specdescription
+
+
+class TestSpecdescription(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        self.open_name = 'specdescription.util.open_auto'
+
+    def setUp(self):
+        specdescription.default_description = "No detailed description available"
+        specdescription.default_description_score = 0
+        specdescription.default_summary = "No detailed summary available"
+        specdescription.default_summary_score = 0
+        specdescription.license.licenses = []
+
+    def test_clean_license_string(self):
+        """
+        Test clean_license_string with a list of license strings
+        """
+        lics = [("ZPL-2.1", "ZPL-2.1"),
+                ("Apache-2.0", "Apache-2.0"),
+                ("GPL-2.0+", "GPL-2.0+"),
+                ("GPL v2", "GPL-2"),
+                ("GPL (>= 2)", "GPL-2.0+"),
+                ("BSD 2-Clause Simplified", "BSD-2-Clause ")]
+
+        for lic, exp in lics:
+            self.assertEqual(specdescription.clean_license_string(lic), exp)
+
+    def test_description_from_spec(self):
+        """
+        Test description_from_spec with summary and description present in
+        spec file
+        """
+        content = "# this is a specfile\n"          \
+                "License: GPL v2\n"                 \
+                "Summary: This is a test package\n" \
+                "\n"                                \
+                "%description\n"                    \
+                "Here is the description section\n" \
+                "Look it continues until the\n"     \
+                "line that begins with '%'\n"       \
+                "\n"                                \
+                "%donewithdesc\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_spec("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "Here is the description section\n"
+                         "Look it continues until the\n"
+                         "line that begins with '%'\n\n")
+        self.assertEqual(specdescription.default_summary,
+                         "This is a test package\n")
+        self.assertEqual(specdescription.default_description_score, 4)
+        self.assertEqual(specdescription.default_summary_score, 4)
+        self.assertEqual(specdescription.license.licenses, ["GPL-2"])
+
+    def test_description_from_spec_no_info(self):
+        """
+        Test description_from_spec with no license, summary or description in
+        spec file
+        """
+        content = "# this is a specfile without much in it\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_spec("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "No detailed description available")
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_description_score, 0)
+        self.assertEqual(specdescription.default_summary_score, 0)
+        self.assertEqual(specdescription.license.licenses, [])
+
+    def test_description_from_pkginfo(self):
+        """
+        Test description_from_pkginfo with information in pkg-info file
+        """
+        content = "# this is a package info file\n"  \
+                "License: GPL v2\n"                  \
+                "abstract: This is a test package\n" \
+                "\n"                                 \
+                "Description:\n"                     \
+                "Here is the description section\n"  \
+                "Look it continues until the\n"      \
+                "line that contains a colon.\n"      \
+                "\n"                                 \
+                "donewithdesc:\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_pkginfo("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "Here is the description section\n"
+                         "Look it continues until the\n"
+                         "line that contains a colon.\n\n")
+        self.assertEqual(specdescription.default_summary,
+                         "This is a test package")
+        self.assertEqual(specdescription.default_description_score, 4)
+        self.assertEqual(specdescription.default_summary_score, 4)
+        self.assertEqual(specdescription.license.licenses, ["GPL-2"])
+
+    def test_description_from_pkginfo_no_info(self):
+        """
+        Test description_from_pkginfo with no information in the pkginfo file
+        """
+        content = "# this is a pkginfo file without much in it\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_pkginfo("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "No detailed description available")
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_description_score, 0)
+        self.assertEqual(specdescription.default_summary_score, 0)
+        self.assertEqual(specdescription.license.licenses, [])
+
+    def test_summary_from_pkgconfig(self):
+        """
+        Test summary_from_pkgconfig with a summary present in the file. The
+        file is named <package>.pc, increasing the summary score
+        """
+        content = "# this is a pkgconfig file\n" \
+                  "Description: this is a test package\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("testpkg.pc", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "this is a test package\n")
+        self.assertEqual(specdescription.default_summary_score, 3)
+
+    def test_summary_from_pkgconfig_misnamed_file(self):
+        """
+        Test summary_from_pkgconfig with a summary present in the file. The
+        file is named pkg.pc, which does not increase the summary score to 3
+        """
+        content = "# this is a pkgconfig file\n" \
+                  "Description: this is a test package\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("pkg.pc", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "this is a test package\n")
+        self.assertEqual(specdescription.default_summary_score, 2)
+
+    def test_summary_from_pkgconfig_no_info(self):
+        """
+        Test summary_from_pkgconfig with no summary in the file.
+        """
+        content = "# this is a pkgconfig file without a description\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("pkg.pc", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_summary_score, 0)
+
+    def test_summary_from_R(self):
+        """
+        Test summary_from_R with DESCRIPTION file
+        """
+        content = "# this is a pkgconfig file\n" \
+                  "Title: this is a test package\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_R("DESCRIPTION")
+
+        self.assertEqual(specdescription.default_summary,
+                         "this is a test package\n")
+        self.assertEqual(specdescription.default_summary_score, 3)
+
+    def test_summary_from_R_no_info(self):
+        """
+        Test summary_from_R with no summary in the file
+        """
+        content = "# this is a DESCRIPTION file without a description\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("DESCRIPTION", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_summary_score, 0)
+
+    def test_skipline(self):
+        """
+        Test skipline for a list of lines
+        """
+        skips = ["Copyright",
+                 "Free Software Foundation, Inc.",
+                 "Copying and distribution of",
+                 "are permitted in any",
+                 "notice and this notice",
+                 "README",
+                 "-*-",
+                 "blabla introduction"]
+
+        for skip in skips:
+            self.assertTrue(specdescription.skipline(skip))
+
+        self.assertFalse(specdescription.skipline('nothing to skip here'))
+
+    def test_description_from_readme(self):
+        """
+        Test description_from_readme with greater than 80 characters and a file
+        name README, increasing the score
+        """
+        content = "This is a readme blah blah blah blah test test test\n" \
+                  "this is the package but it is just a test package\n"   \
+                  "ok?\n\n"                                               \
+                  "This is a new paragraph, shouldn't be included"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_readme("README")
+
+        self.assertEqual(specdescription.default_description,
+                         "This is a readme blah blah blah blah test test test\n"
+                         "this is the package but it is just a test package\n"
+                         "ok?\n")
+        self.assertEqual(specdescription.default_description_score, 1.5)
+
+    def test_description_from_readme_with_skiplines(self):
+        """
+        Test description_from_readme with a skipline.
+        """
+        content = "This is a readme blah blah blah blah test test test\n"  \
+                  "this is the package Copying and distribution of this\n" \
+                  "ok? More characters needed to get over 80\n\n"          \
+                  "This is a new paragraph, shouldn't be included"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_readme("Readme.md")
+
+        self.assertEqual(specdescription.default_description,
+                         "This is a readme blah blah blah blah test test test\n"
+                         "ok? More characters needed to get over 80\n")
+        self.assertEqual(specdescription.default_description_score, 1)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_specfile.py

@@ -0,0 +1,457 @@
+import unittest
+import unittest.mock
+import buildreq
+import config
+import specfiles
+import tarball
+
+
+class TestSpecfileWrite(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        self.specfile = None
+        self.WRITES = []
+
+    def setUp(self):
+        conf = config.Config('/download/path')
+        conf.config_opts['dev_requires_extras'] = False
+        url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz"
+        content = tarball.Content(url, 'pkg', '1.0', [], conf, '/tmp')
+        conf.content = content
+        reqs = buildreq.Requirements(url)
+        self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf, reqs, content)
+
+        def mock_write(string):
+            self.WRITES.append(string)
+
+        self.specfile._write = mock_write
+        self.specfile._write_strip = mock_write
+        self.WRITES = []
+
+    def test_write_comment_header(self):
+        """
+        test Specfile.write_comment_header with all optional fields
+        """
+        self.specfile.keyid = '1'
+        self.specfile.email = 'email'
+        self.specfile.write_comment_header()
+        expect = ["#\n",
+                  "# This file is auto-generated. DO NOT EDIT\n",
+                  "# Generated by: autospec.py\n",
+                  "# Using build pattern: make\n",
+                  "#\n",
+                  "# Source0 file verified with key 0x1 (email)",
+                  "#"]
+        # skipping autospec version lines as they change
+        self.WRITES = self.WRITES[:4] + self.WRITES[6:]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_nvr(self):
+        """
+        test Specfile.write_nvr with no urlban set
+        """
+        self.specfile.write_nvr()
+        expect = ["Name     : pkg\n",
+                  "Version  : 1.0\n",
+                  "Release  : 2\n",
+                  "URL      : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n",
+                  "Source0  : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_sources(self):
+        """
+        test write_sources with all Specfile.sources set.
+        """
+        self.specfile.config.sources["unit"] = ["pkg2.service", "pkg1.service"]
+        self.specfile.config.sources["archive"] = ["archA", "archD", "archB", "archC"]
+        self.specfile.config.sources["tmpfile"] = ["tmp1", "tmp2"]
+        self.specfile.config.sources["gcov"] = ["pkg.gcov"]
+        self.specfile.write_sources()
+        expect = ["Source1  : archA\n",
+                  "Source2  : archB\n",
+                  "Source3  : archC\n",
+                  "Source4  : archD\n",
+                  "Source5  : pkg.gcov\n",
+                  "Source6  : pkg1.service\n",
+                  "Source7  : pkg2.service\n",
+                  "Source8  : tmp1\n",
+                  "Source9  : tmp2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_summary(self):
+        """
+        test write_summary with unstripped summary and group strings
+        """
+        self.specfile.default_sum = "   This is unstripped summary  "
+        self.specfile.write_summary()
+        expect = ["Summary  : This is unstripped summary\n",
+                  "Group    : Development/Tools\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_license(self):
+        """
+        test write_license with unsorted list of licenses
+        """
+        self.specfile.licenses = ["MIT", "IJG", "GPL-3.0", "GPL-2.0", "ICU"]
+        self.specfile.write_license()
+        expect = ["License  : GPL-2.0 GPL-3.0 ICU IJG MIT\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_main_subpackage_requires_no_autostart(self):
+        """
+        test write_main_subpackage_requires with autostart package and
+        Specfile.no_autostart set.
+        """
+        self.specfile.packages["autostart"] = ["autostart"]
+        self.specfile.packages["bin"] = []
+        self.specfile.packages["lib"] = ["package.so"]
+        self.specfile.requirements.requires[None] = set(["pkg1", "pkg2"])
+        self.specfile.config.config_opts['no_autostart'] = True
+        self.specfile.write_main_subpackage_requires()
+        expect = ["Requires: pkg-bin = %{version}-%{release}\n",
+                  "Requires: pkg-lib = %{version}-%{release}\n",
+                  "Requires: pkg1\n",
+                  "Requires: pkg2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_main_subpackage_requires_autostart(self):
+        """
+        test write_main_subpackage_requires with autostart package and
+        Specfile.no_autostart not set. Also has other ignored packages.
+        """
+        self.specfile.packages["autostart"] = ["autostart"]
+        self.specfile.packages["bin"] = []
+        self.specfile.packages["lib"] = ["package.so"]
+        self.specfile.packages["main"] = ["main/file.py"]
+        self.specfile.packages["ignore"] = []
+        self.specfile.packages["dev"] = []
+        self.specfile.packages["active-units"] = []
+        self.specfile.requirements.requires[None] = set(["pkg1", "pkg2"])
+        self.specfile.write_main_subpackage_requires()
+        expect = ["Requires: pkg-autostart = %{version}-%{release}\n",
+                  "Requires: pkg-bin = %{version}-%{release}\n",
+                  "Requires: pkg-lib = %{version}-%{release}\n",
+                  "Requires: pkg1\n",
+                  "Requires: pkg2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_main_subpackage_requires_empty(self):
+        """
+        test write_main_subpackage_requires with no required subpackages.
+        """
+        self.specfile.write_main_subpackage_requires()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_buildreq(self):
+        """
+        test write_buildreq with unsorted list of build requirements.
+        """
+        self.specfile.requirements.buildreqs = ["python", "ruby", "go"]
+        self.specfile.write_buildreq()
+        expect = ["BuildRequires : go\n",
+                  "BuildRequires : python\n",
+                  "BuildRequires : ruby\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_buildreq_empty(self):
+        """
+        test write_buildreq with no build requirements.
+        """
+        self.specfile.write_buildreq()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_patch_header(self):
+        """
+        test write_patch_header with list of patches.
+        """
+        self.specfile.config.patches = ["speedup.patch",
+                                        "slowdown.patch",
+                                        "revert.patch reverts slowdown.patch"]
+        self.specfile.write_patch_header()
+        expect = ["Patch1: speedup.patch\n",
+                  "Patch2: slowdown.patch\n",
+                  "Patch3: revert.patch\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_patch_header_empty(self):
+        """
+        test write_patch_header with empty patch list.
+        """
+        self.specfile.write_patch_header()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_description(self):
+        """
+        test write_description with unstripped description
+        """
+        self.specfile.default_desc = " This package is for testing only       "
+        self.specfile.write_description()
+        expect = ["\n%description\nThis package is for testing only\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_header_base(self):
+        """
+        base test for write_files_header
+        """
+        self.specfile.packages["data"] = ["df1", "df2", "df3"]
+        self.specfile.packages["ignore"] = ["doesn't matter"]
+        self.specfile.packages["python"] = ["pyfile1", "pyfile2"]
+        self.specfile.packages["dev"] = ["dev1", "dev2"]
+        self.specfile.packages["pack"] = ["packf1"]
+        self.specfile.requirements.buildreqs = ["pep8", "pycurl"]
+        self.specfile.write_files_header()
+        expect = ["\n%package data\n",
+                  "Summary: data components for the pkg package.\n",
+                  "Group: Data\n",
+                  "\n%description data\n",
+                  "data components for the pkg package.\n",
+                  "\n",
+                  "\n%package dev\n",
+                  "Summary: dev components for the pkg package.\n",
+                  "Group: Development\n",
+                  "Requires: pkg-data = %{version}-%{release}\n",
+                  "Provides: pkg-devel = %{version}-%{release}\n",
+                  "Requires: pkg = %{version}-%{release}\n",
+                  "\n%description dev\n",
+                  "dev components for the pkg package.\n",
+                  "\n",
+                  "\n%package pack\n",
+                  "Summary: pack components for the pkg package.\n",
+                  "Group: Default\n",
+                  "\n%description pack\n",
+                  "pack components for the pkg package.\n",
+                  "\n",
+                  "\n%package python\n",
+                  "Summary: python components for the pkg package.\n",
+                  "Group: Default\n",
+                  "\n%description python\n",
+                  "python components for the pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_header_custom_extra_requires(self):
+        """
+        test write_files_header with custom extras requires.
+        """
+        self.specfile.packages["data"] = ["file1"]
+        self.specfile.packages["foobar-extras"] = ["file2"]
+        self.specfile.file_maps = { 'foobar-extras': { 'requires': ["data"] }}
+        self.specfile.write_files_header()
+        expect = ["\n%package data\n",
+                  "Summary: data components for the pkg package.\n",
+                  "Group: Data\n",
+                  "\n%description data\n",
+                  "data components for the pkg package.\n",
+                  "\n",
+                  "\n%package foobar-extras\n",
+                  "Summary: foobar-extras components for the pkg package.\n",
+                  "Group: Default\n",
+                  "Requires: pkg-data = %{version}-%{release}\n",
+                  "\n%description foobar-extras\n",
+                  "foobar-extras components for the pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_header_python_name(self):
+        """
+        test write_files_header with uppercase letter in Specfile.name, causing
+        a Provides line to be written.
+        """
+        self.specfile.name = 'Pkg'
+        self.specfile.packages["python"] = ["pyfile1", "pyfile2"]
+        self.specfile.write_files_header()
+        expect = ["\n%package python\n",
+                  "Summary: python components for the Pkg package.\n",
+                  "Group: Default\n",
+                  "Provides: pkg-python\n",
+                  "\n%description python\n",
+                  "python components for the Pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
+
+    def test_write_files_header_bare(self):
+        """
+        test write_files_header with no packages
+        """
+        self.specfile.write_files_header()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_buildpattern(self):
+        """
+        test write_buildpattern
+        """
+        # Should be doable but probably worth waiting for specfile template rework
+        pass
+
+    def test_write_scriplets_base(self):
+        """
+        test write_scriplets base test
+        """
+        def mock_read_conf_file(name):
+            prefix = "pre" if "pre" in name else "post"
+            return ["{}-script line 1\n".format(prefix),
+                    "{}-script line 2\n".format(prefix),
+                    "{}-script line 3\n".format(prefix)]
+
+        class MockSpecfile(object):
+            @staticmethod
+            def writelines(stringlist):
+                self.WRITES.extend(stringlist)
+
+        self.specfile.config.read_conf_file = mock_read_conf_file
+        self.specfile.specfile = MockSpecfile()
+        self.specfile.packages["ruby"] = ["rubyfile1"]
+        self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
+        self.specfile.write_scriplets()
+        expect = ["\n%post ruby\n",
+                  "post-script line 1\n\n",
+                  "post-script line 2\n\n",
+                  "post-script line 3\n\n",
+                  "\n%pre ruby\n",
+                  "pre-script line 1\n\n",
+                  "pre-script line 2\n\n",
+                  "pre-script line 3\n\n"]
+
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_scriplets_pre_only(self):
+        """
+        test write_scriplets with only pre-scripts present.
+        """
+        def mock_read_conf_file(name):
+            if "pre" in name:
+                return ["pre-script line 1\n",
+                        "pre-script line 2\n",
+                        "pre-script line 3\n"]
+            else:
+                return []
+
+        class MockSpecfile(object):
+            @staticmethod
+            def writelines(stringlist):
+                self.WRITES.extend(stringlist)
+
+        self.specfile.config.read_conf_file = mock_read_conf_file
+        self.specfile.specfile = MockSpecfile()
+        self.specfile.packages["ruby"] = ["rubyfile1"]
+        self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
+        self.specfile.write_scriplets()
+        expect = ["\n%pre ruby\n",
+                  "pre-script line 1\n\n",
+                  "pre-script line 2\n\n",
+                  "pre-script line 3\n\n"]
+
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_scriplets_bare(self):
+        """
+        test write_scriplets with no scripts present.
+        """
+        def mock_read_conf_file(name):
+            return []
+
+        class MockSpecfile(object):
+            @staticmethod
+            def writelines(stringlist):
+                self.WRITES.extend(stringlist)
+
+        self.specfile.config.read_conf_file = mock_read_conf_file
+        self.specfile.specfile = MockSpecfile()
+        self.specfile.packages["ruby"] = ["rubyfile1"]
+        self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
+        self.specfile.write_scriplets()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_files_base(self):
+        """
+        test write_files base test.
+        """
+        self.specfile.packages["main"] = ["mainfile1", "/mainfile2", "/mainfile3",
+                "/mainfile 4", "mainfile\t5", "%foo /mainfile6", "%bar /mainfile 7"]
+        self.specfile.packages["ignore"] = ["ignorepkg"]
+        self.specfile.packages["other"] = ["other2", "other1"]
+        self.specfile.write_files()
+        # Note the special sorting
+        expect = ["\n%files\n",
+                  "%defattr(-,root,root,-)\n",
+                  '%bar "/mainfile 7"\n',
+                  "%foo /mainfile6\n",
+                  '"/mainfile 4"\n',
+                  "/mainfile2\n",
+                  "/mainfile3\n",
+                  '"mainfile\t5"\n',
+                  "mainfile1\n",
+                  "\n%files other\n",
+                  "%defattr(-,root,root,-)\n",
+                  "other1\n",
+                  "other2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_empty(self):
+        """
+        test write_files with empty package list.
+        """
+        self.specfile.write_files()
+        expect = ["\n%files\n",
+                  "%defattr(-,root,root,-)\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_lang_files_base(self):
+        """
+        write_lang_files base test
+        """
+        self.specfile.locales = ["pkg1", "pkg2"]
+        self.specfile.write_lang_files()
+        expect = ["\n%files locales",
+                  " -f pkg1.lang",
+                  " -f pkg2.lang",
+                  "\n%defattr(-,root,root,-)\n\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_lang_files_empty(self):
+        """
+        test write_lang_files with empty lang list
+        """
+        self.specfile.write_lang_files()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_service_restart(self):
+        """
+        Validate that service_restart configuration is written to the spec file
+        correctly.
+        """
+        self.specfile.config.service_restart = [
+                "/usr/lib/systemd/system/foo.service",
+                "/usr/lib/systemd/system/bar.service",
+        ]
+        self.specfile.write_service_restart()
+        expect = [
+            "## service_restart content",
+            "mkdir -p %{buildroot}/usr/share/clr-service-restart",
+            "ln -s /usr/lib/systemd/system/foo.service %{buildroot}/usr/share/clr-service-restart/foo.service",
+            "ln -s /usr/lib/systemd/system/bar.service %{buildroot}/usr/share/clr-service-restart/bar.service",
+            "## service_restart end",
+        ]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_exclude_deletes(self):
+        """
+        Validate that excludes configuration is written to the spec file correctly.
+        """
+        self.specfile.excludes = [
+                "/usr/bin/bar",
+                "/usr/bin/foo",
+        ]
+        self.specfile.write_exclude_deletes()
+        expect = [
+            "## Remove excluded files",
+            "rm -f %{buildroot}*/usr/bin/bar",
+            "rm -f %{buildroot}*/usr/bin/foo",
+        ]
+        self.assertEqual(expect, self.WRITES)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_tarball.py

@@ -0,0 +1,236 @@
+import copy
+import os
+import unittest
+from collections import OrderedDict
+from unittest.mock import MagicMock, Mock, patch
+import build
+import config
+import files
+import tarball
+
+
+# Stores all test cases for dynamic tests.
+# In order to add more tests just add more elements to the lists provided below.
+
+CONTENT_PECL = [
+    'package.xml',
+    'common-prefix/',
+    'common-prefix/md5/',
+    'common-prefix/md5/CMakeLists.txt',
+    'common-prefix/md5/md5.h',
+    'common-prefix/md5/md5hl.c',
+    'common-prefix/md5/md5cmp.c',
+    'common-prefix/md5/md5.c',
+    'common-prefix/md5/Makefile.am',
+    'common-prefix/md5/Makefile.in',
+    'common-prefix/jerror.c',
+    'common-prefix/sharedlib/',
+    'common-prefix/sharedlib/CMakeLists.txt',
+    'common-prefix/turbojpeg-mapfile',
+    'common-prefix/jdpostct.c',
+    'common-prefix/turbojpeg-jni.c',
+]
+
+CONTENT_PREFIX = [
+    'common-prefix/',
+    'common-prefix/md5/',
+    'common-prefix/md5/CMakeLists.txt',
+    'common-prefix/md5/md5.h',
+    'common-prefix/md5/md5hl.c',
+    'common-prefix/md5/md5cmp.c',
+    'common-prefix/md5/md5.c',
+    'common-prefix/md5/Makefile.am',
+    'common-prefix/md5/Makefile.in',
+    'common-prefix/jerror.c',
+    'common-prefix/sharedlib/',
+    'common-prefix/sharedlib/CMakeLists.txt',
+    'common-prefix/turbojpeg-mapfile',
+    'common-prefix/jdpostct.c',
+    'common-prefix/turbojpeg-jni.c',
+]
+
+CONTENT_SUBDIR = [
+    'dir1/',
+    'dir1/md5/',
+    'dir1/md5/CMakeLists.txt',
+    'dir1/md5/md5.h',
+    'dir1/md5/md5hl.c',
+    'dir1/md5/md5cmp.c',
+    'dir1/md5/md5.c',
+    'dir1/md5/Makefile.am',
+    'dir1/md5/Makefile.in',
+    'dir2/',
+    'dir2/jerror.c',
+    'dir2/sharedlib/',
+    'dir2/sharedlib/CMakeLists.txt',
+    'dir2/turbojpeg-mapfile',
+    'dir2/jdpostct.c',
+    'dir2/turbojpeg-jni.c',
+    'file.c'
+]
+
+# Input for tarball.Source class tests.
+# Structure: (url, destination, path, fake-content, source_type, prefix, subddir)
+SRC_CREATION = [
+    ("https://example/src-PECL.tar", "", "/tmp/src-PECL.tar", CONTENT_PECL, "tar", "common-prefix", None),
+    ("https://example/src-non-PECL.tar", "", "/tmp/src-non-PECL.tar", CONTENT_PECL, "tar", "", "src-non-PECL"),
+    ("https://example/src-prefix.zip", "", "/tmp/src-prefix.zip", CONTENT_PREFIX, "zip", "common-prefix", None),
+    ("https://example/src-subdir.zip", "", "/tmp/src-subdir.zip", CONTENT_SUBDIR, "zip", "", "src-subdir"),
+    ("https://example/src-prefix.tar", "", "/tmp/src-prefix.tar", CONTENT_PREFIX, "tar", "common-prefix", None),
+    ("https://example/src-subdir.tar", "", "/tmp/src-subdir.tar", CONTENT_SUBDIR, "tar", "", "src-subdir"),
+    ("https://example/src-no-extractable.tar", ":", "/tmp/src-no-extractable.tar", None, None, None, None),
+]
+
+
+class MockSrcFile():
+    """Mock class for zipfile and tarfile."""
+
+    def __init__(self, path, mode):
+        self.name = path
+        self.mode = mode
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, traceback):
+        return False
+
+    @classmethod
+    def set_content(cls, content):
+        # deep copy because the content is modified by Source
+        cls.content = copy.deepcopy(content)
+
+    def getnames(self):
+        return self.content
+
+    def namelist(self):
+        return self.content
+
+
+def source_test_generator(url, destination, path, content, src_type, prefix, subdir):
+    """Create test for tarball.Source class using generator template."""
+
+    @patch('tarball.tarfile.open', MockSrcFile)
+    @patch('tarball.zipfile.ZipFile', MockSrcFile)
+    @patch('tarball.tarfile.is_tarfile', Mock(return_value=True))
+    @patch('tarball.zipfile.is_zipfile', Mock(return_value=True))
+    def generator(self):
+        """Test template."""
+        # Set fake content
+        MockSrcFile.set_content(content)
+        if os.path.basename(path) in ['src-PECL.tar']:
+            src = tarball.Source(url, destination, path, 'phpize')
+        else:
+            src = tarball.Source(url, destination, path)
+        self.assertEqual(src.type, src_type)
+        self.assertEqual(src.prefix, prefix, f"fail for: {url}")
+        self.assertEqual(src.subdir, subdir)
+
+    return generator
+
+
+def name_and_version_test_generator(url, name, version):
+    """Create test for tarball.name_and_version method."""
+    def generator(self):
+        """Test template."""
+        conf = config.Config('/download/path')
+        conf.parse_config_versions = Mock(return_value={})
+        # Test four different name/version states for tarball.Content, each in
+        # a subtest. Test failures will print these state numbers for easy
+        # identification:
+        # 0 - no state
+        # 1 - name only
+        # 2 - version only
+        # 3 - name and version
+        for state in range(4):
+            with self.subTest(state=state):
+                name_arg = ""
+                version_arg = ""
+                if state == 1 or state == 3:
+                    name_arg = f"state.{name}"
+                if state == 2 or state == 3:
+                    version_arg = f"state.{version}"
+                content = tarball.Content(url, name_arg, version_arg, [], conf, '/tmp')
+                content.config = conf
+                pkg = build.Build()
+                mgr = files.FileManager(conf, pkg)
+                content.name_and_version(mgr)
+                name_cmp = name
+                version_cmp = version
+                if state == 1 or state == 3:
+                    name_cmp = name_arg
+                if state == 2 or state == 3:
+                    version_cmp = version_arg
+                self.assertEqual(name_cmp, content.name)
+                self.assertEqual(version_cmp, content.version)
+        # redo without args and verify giturl is set correctly
+        content.name = ""
+        content.version = ""
+        content.name_and_version(Mock())
+        if "github.com" in url:
+            self.assertRegex(content.giturl, r"https://github.com/[^/]+/" + content.repo + ".git")
+
+    return generator
+
+
+def create_dynamic_tests():
+    """Create dynamic tests based on content in lists and packageulrs file."""
+    # Create tests for tarball.Source class.
+    for url, dest, path, content, src_type, prefix, subdir in SRC_CREATION:
+        test_name = 'test_src_{}'.format(url)
+        test = source_test_generator(url, dest, path, content, src_type, prefix, subdir)
+        setattr(TestTarball, test_name, test)
+
+    # Create tests for tarball.name_and_version method.
+    with open('tests/packageurls', 'r') as pkgurls:
+        for urlline in pkgurls.read().split('\n'):
+            if not urlline or urlline.startswith('#'):
+                continue
+            (url, name, version) = urlline.split(',')
+            test_name = 'test_name_ver_{}'.format(url)
+            test = name_and_version_test_generator(url, name, version)
+            setattr(TestTarball, test_name, test)
+
+
+class TestTarball(unittest.TestCase):
+    """Main testing class for tarball.py.
+
+    This class would contain all static tests and dynamic tests for tarball.py
+    """
+
+    def setUp(self):
+        """Set up default values before start test."""
+        # Set strenght to 0 so it can be updated during tests
+        conf = config.Config('/download/path')
+        self.content = tarball.Content('', '', '', [], conf, '/tmp')
+        conf.content = self.content
+
+    @patch('tarball.os.path.isfile', Mock(return_value=True))
+    def test_set_gcov(self):
+        """Test for tarball.set_gcov method."""
+        # Set up input values
+        self.content.name = 'test'
+        self.content.set_gcov()
+        self.assertEqual(self.content.gcov_file, 'test.gcov')
+
+    @patch('tarball.Source.set_prefix', Mock())
+    @patch('tarball.Source.extract', Mock())
+    def test_extract_sources(self):
+        """Test for Content extract_sources method."""
+        # Set up input values
+        main_src = tarball.Source('https://example1.tar', '', '/tmp')
+        arch1_src = tarball.Source('https://example2.tar', '', '/tmp')
+        arch2_src = tarball.Source('https://example3.tar', ':', '/tmp')
+        arch3_src = tarball.Source('https://example4.tar', '', '/tmp')
+        archives_src = [arch1_src, arch2_src, arch3_src]
+        self.content.extract_sources(main_src, archives_src)
+        # Sources with destination=':' should not be extracted, so method
+        # should be called only 3 times.
+        self.assertEqual(tarball.Source.extract.call_count, 3)
+
+
+# Create dynamic tests based on config file
+create_dynamic_tests()
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_util.py

@@ -0,0 +1,135 @@
+import subprocess
+import os
+import tempfile
+import unittest
+from unittest.mock import MagicMock, mock_open, patch
+
+import util
+
+
+def mock_gen(rv=None):
+    def mock_f(*args, **kwargs):
+        return rv
+
+    return mock_f
+
+
+class TestUtil(unittest.TestCase):
+
+    def test_call(self):
+        """
+        Test call with default arguments, make sure it passes out the correct
+        returncode
+        """
+        call_backup = subprocess.call
+        util.subprocess.call = mock_gen(rv=0)
+        self.assertEqual(util.call('some command'), 0)
+        util.subprocess.call = call_backup
+
+    def test_call_check(self):
+        """
+        Test call with check=True (default) and a bad returncode. Should raise a
+        CalledProcessError
+        """
+        call_backup = subprocess.call
+        util.subprocess.call = mock_gen(rv=1)
+        with self.assertRaises(subprocess.CalledProcessError):
+            util.call('some command')
+
+        util.subprocess.call = call_backup
+
+    def test_call_no_check(self):
+        """
+        Test call with check=False and a bad returncode, should return the
+        returncode
+        """
+        call_backup = subprocess.call
+        util.subprocess.call = mock_gen(rv=1)
+        self.assertEqual(util.call('some command', check=False), 1)
+        util.subprocess.call = call_backup
+
+    def test_translate(self):
+        """
+        Spot-test the translate function with a package defined in
+        translate.dic
+        """
+        self.assertEqual(util.translate('dateutil-python'), 'pypi-python_dateutil')
+
+    def test_binary_in_path(self):
+        """
+        Test binary_in_path
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            open(os.path.join(tmpd, 'testbin'), 'w').close()
+            util.os.environ["PATH"] = tmpd
+            self.assertTrue(util.binary_in_path('testbin'))
+            self.assertEqual(util.os_paths, [tmpd])
+
+    def test__process_build_log_bad_patch(self):
+        """
+        Test _process_build_log with a bad patch
+        """
+        def isfile_mock(_):
+            return True
+        isfile_backup = util.os.path.isfile
+        util.os.path.isfile = isfile_mock
+        call_backup = util.call
+        util.call = MagicMock()
+        open_name = 'util.open_auto'
+        content = "Patch #1 (bad.patch):\nHunk #1 FAILED at 1."
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            util._process_build_log('filename')
+
+        util.os.path.isfile = isfile_backup
+        mock_call = util.call
+        util.call = call_backup
+        self.assertTrue(len(mock_call.mock_calls) == 3)
+
+    def test_globlike_match(self):
+        """
+        Test globlike_match
+        """
+        match_name = ['a', 'b', 'c']
+        file_path = 'a/b'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', 'c']
+        file_path = 'a/b'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', 'bb*']
+        file_path = 'a/b'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', 'b*']
+        file_path = 'a/ab'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', '*a']
+        file_path = 'a/ab'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', 'c*']
+        file_path = 'a/b'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', '*c']
+        file_path = 'a/b'
+        self.assertFalse(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', 'b*']
+        file_path = 'a/b'
+        self.assertTrue(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', '*b']
+        file_path = 'a/b'
+        self.assertTrue(util.globlike_match(file_path, match_name))
+
+        match_name = ['a', 'b']
+        file_path = 'a/b'
+        self.assertTrue(util.globlike_match(file_path, match_name))
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/testfiles/pkg_integrity/pkg-config-0.29.1.tar.gz.asc

@@ -0,0 +1,17 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJW2ZtDAAoJEAI6RCDH7GkUc0IP/1bH7KEJdUM+lrGM1SOuNHdq
+4VEwDp1II8abbBzHeGEXZ8p4+MwwSOYHFiy+NM1yldZkDXtqAlAqvIuEzc+PtgGd
+vFeNPb9infibNaEDK+zz4fcqJOSab1ZcQ/D3EIJXwKr5nIYP8RuCHu/zstf7o6R0
+/wnGWaAIB1+p9PxvUhMPMbBEQCw/cBzyZ2d7nApHF3b0OH2wM7P8VG8ot4cuglPq
+hzk27ZnrYeUDyUUhMRlL7sZZouJlSy/0OxsBK++tOjE6MiuAZhqtlSW+cFK4L7k/
+q4eLodX7GtF0psSgTRjTk2ozdSIDkB2ccLBN6CzgCcbPrbcz4tVQqaQBcSd0mCl7
+RWAKmSye7p+CY8mIIOjdYm+KaQRmJMKDXs49hMycti22jnu5T2BM6O7MZpiY+cb3
+O2UKUXbVyX/cXKwTYwf4VMddxJKFaqYac+7n5qWbdwBjk9E5OC2ltz94taM1pxZ5
+2jRtfyIb3s+Rj6M5cXI5UChrGqzMK6BmEbyZ0KbHAJ7Y0xvGqwydC6J+RwGIqRlp
+LmW3k1ggpUajoMcgq9KqJgVqo/9f4+6anADHRMNJ93MxR7h5BRQ1/GWSXpOzYsYv
+DnfQPrhc+z20m81qsvfUcBnN/k74yiDVqp3I/HrYGD+f8cXKPpBRESkPAXlUIcu2
+ALHZXEdBGcWdUrAIpJT2
+=SMbV
+-----END PGP SIGNATURE-----

tests/testfiles/pkg_integrity/quagga-1.1.0.tar.gz.asc

@@ -0,0 +1,18 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQJMBAABCAA2BQJYBh/XLxpodHRwczovL3d3dy5qYWttYS5vcmcvfnBhdWwvcGdw
+X3BvbGljeS0xLjEudHh0AAoJEG/lfKjBpK6mNOMP/irlUcU/4NQ6BQQbF7Vc90wo
+hmTH/zGAodnndxSplIGV63BQqmQr5KiSOp6tNQ3OlCIwLFWlr/LiJz+MeThaMtZx
+/mlrZptWIhbEcJzVa6efg8UGIYl2NC+QxFgEyezfCYSEmczd1qE3kiX+5WjcqVPH
+pVjisPwaAYa0FTCsAwBMUKyJr2K3sq5hSJR7DFDVnvH1DJ1xZd9871ZdDpZBHz2z
+GvR+IuZcWbYleH1ArFvT6cMokTpIUMvd33/+Gdpfu9fihzQQn199nN2LFZJzuEGV
+vH6+IZVTmtXb5U/sdtbWGaDv8eFLAWl2NH9VNdlVw5FbWOYRC4YNN39/hBy7s0po
+hvq33ZugC7JqPuje+4W1oF1T/dbRCIBmWUzKsoH8+3z4KJHdotSR0cU+TT+w6SJC
+QhF9TAbjrfbeJs0D0NnZrllDLiLGFgLl5yULzMjRDqKcgqE5+nvYBPXHPUCCPU7+
+59QOkPMsz/kZGV1lRzoUoxlM6V/phJRPU7jit10puiNij0c4peWbjVmTWDei3Xeu
+kHpck5wIAUFzAIXBUpVhYdLl/kxq654Qqthoci869ATZIKH/SOId9Y0K1GE6xO6H
+7vQFUGn3dcdnbgapM+tpmC77EmV4BtzkTjcu2pX6s0qnI4P+M+zXS+7G96xP3zHN
+ZSFq45swEKd6SKbOYxcN
+=mt1F
+-----END PGP SIGNATURE-----