Limit diff to ~10 lines

Also include content of added files.

.github/actions/clearlinux-latest-action/Dockerfile

@@ -0,0 +1,4 @@
+FROM clearlinux:latest
+RUN swupd bundle-add package-builder python-extras
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/clearlinux-latest-action/action.yml

@@ -0,0 +1,10 @@
+name: 'Clear in Docker'
+description: 'Run commands in the latest Clear Linux OS Docker image'
+inputs:
+  testfunc:
+    description: 'Test function to run'
+    required: true
+    default: 'run_flake8'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/clearlinux-latest-action/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -lx
+
+run_flake8() {
+	make check
+}
+
+run_unittests() {
+	make unittests-no-coverage
+}
+
+if t=$(type -t "$INPUT_TESTFUNC"); then
+	if [ "$t" = "function" ]; then
+		$INPUT_TESTFUNC
+	fi
+fi

.github/workflows/tests.yml

@@ -0,0 +1,27 @@
+name: Autospec Tests
+
+on: [push, pull_request]
+
+jobs:
+  test_style:
+    runs-on: ubuntu-latest
+    name: Flake8
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.3.0
+      - name: Run Tests
+        uses: ./.github/actions/clearlinux-latest-action
+        with:
+          testfunc: run_flake8
+  test_unit:
+    runs-on: ubuntu-latest
+    name: Unit
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.3.0
+      - name: Run Tests
+        uses: ./.github/actions/clearlinux-latest-action
+        with:
+          testfunc: run_unittests

.gitignore

@@ -1,2 +1,7 @@
 __pycache__
 *.pyc
+*~
+*.swp
+tags
+.coverage
+htmlcov

Makefile

@@ -1,2 +1,59 @@
-check: *.py
+check: autospec/*.py
-	@python3 /usr/bin/flake8 --max-line-length=199 --ignore=E402 $^
+	@flake8 --ignore=B902,D100,I201 $^
+
+test_download:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_download.py
+
+test_pkg_integrity:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_pkg_integrity.py
+
+test_tarball:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_tarball.py
+
+test_specfile:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_specfile.py
+
+test_abireport:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_abireport.py
+
+test_commitmessage:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_commitmessage.py
+
+test_files:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_files.py
+
+test_license:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_license.py
+
+test_config:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_config.py
+
+test_build:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_build.py
+
+test_buildreq:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_buildreq.py
+
+test_specdescription:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_specdescription.py
+
+test_count:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_count.py
+
+test_check:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_check.py
+
+test_util:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_util.py
+
+test_general:
+	PYTHONPATH=${CURDIR}/autospec python3 tests/test_general.py
+
+unittests:
+	PYTHONPATH=${CURDIR}/autospec coverage run -m unittest discover -b -s tests -p 'test_*.py' && coverage report
+
+unittests-no-coverage:
+	PYTHONPATH=${CURDIR}/autospec python3 -m unittest discover -b -s tests -p 'test_*.py'
+
+coverage:
+	coverage report -m

README.rst

@@ -2,23 +2,25 @@
 Autospec
 ========
 
-autospec is a tool to assist in the automated creation and maintainence
+autospec is a tool to assist in the automated creation and maintenance of RPM
-of RPM packaging. It will continuously run updated builds based on new
+packaging. It will continuously run updated builds based on new information
-information discovered from build failures, until it has a complete and
+discovered from build failures until it has a complete and valid .spec file. The
-valid .spec file. The tool makes use of mock to achieve this.
+tool makes use of mock to achieve this.
+
+.. contents:: Table of Contents
 
 License
 =======
 autospec is available under the terms of the GPL, version 3.0
 
-Copyright (C) 2015 Intel Corporation
+Copyright (C) 2017 Intel Corporation
 
 
 Configuration of autospec
 =========================
 autospec is configured by means of a simple INI-style configuration file.
-The default location of this file is assumed to be ``common/autospec.conf``,
+The default location of this file is assumed to be
-relative to the directory in which autospec is executed.
+``/usr/share/defaults/autospec/autospec.conf``.
 
 Example ``autospec.conf`` file::
 
@@ -26,40 +28,67 @@ Example ``autospec.conf`` file::
     git = git@someurl.com/%(NAME)s
     license_fetch = http://yourhost/hash.php
     license_show = http://yourhost/showone.php?hash=%(HASH)s
+    packages_file = file:///path/to/package_list_file
+    yum_conf = file:///path/to/yum.conf
     upstream = http://yourhost/tarballs/%(HASH)s/%(NAME)s
 
+git
+  Optional URI template for remote git repository
 
-**git**
+license_fetch
-    The upstream git repository URL base
+  Optional URL to use for scanning license files
 
-**license_fetch**
+license_show
-    Optional URL to use for scanning license files
+  Optional URL to interact with online license checker
 
-**license_show**
+packages_file
-    Optional URL to interact with online license checker
+  Optional path to add autodetected runtime requirement checking
 
-**upstream**
+yum_conf
-    Base URL for stored upstream tarballs
+  Optional path to yum configuration
+
+upstream
+  Base URL for stored upstream tarballs
 
 Synopsis
 ========
 
-Usage: ``python3 autospec.py [options] URL``
+.. code-block:: bash
 
+  usage: autospec.py [-h] [-g] [-n NAME] [-v VERSION]
+                     [-a [ARCHIVES [ARCHIVES ...]]] [-l] [-b] [-c CONFIG]
+                     [-t TARGET] [-i] [-p] [--non_interactive] [-C]
+                     [-m MOCK_CONFIG] <url>
 
--h, --help                                      show help message and exit
+    url                 (required) tarball URL
--g, --skip-git                                  Don't commit result to git
+                        (e.g. http://example.com/downloads/mytar.tar.gz)
--n NAME, --name NAME                            Override the package name
--a ARCHIVES, --archives ARCHIVES
-                                                tarball URLs for additional source archives and a
-                                                location for the sources to be extacted to (e.g.
-                                                http://example.com/downloads/dependency.tar.gz
-                                                /directory/relative/to/extract/root )
--l, --license-only                              Only scan for license files
--b, --skip-bump                                 Don't bump release number
--c CONFIG, --config CONFIG                      Set configuration file to use
--t DIRECTORY --target DIRECTORY                 Set location to create or use
 
+optional arguments:
+  -h, --help            show this help message and exit
+  -g, --skip-git        Don't commit result to git
+  -n NAME, --name NAME  Override the package name
+  -v VERSION, --version VERSION
+                        Override the package version
+  -a ARCHIVES, --archives ARCHIVES
+                        tarball URLs for additional source archives and a
+                        location for the sources to be extacted to (e.g.
+                        http://example.com/downloads/dependency.tar.gz
+                        /directory/relative/to/extract/root )
+  -l, --license-only    Only scan for license files
+  -b, --skip-bump       Don't bump release number
+  -c CONFIG, --config CONFIG
+                        Set configuration file to use
+  -t TARGET, --target TARGET
+                        Target location to create or reuse
+  -i, --integrity       Search for package signature from source URL and
+                        attempt to verify package
+  -p, --prep-only       Only perform preparatory work on package
+  --non_interactive     Disable interactive mode for package verification
+  -C, --cleanup         Clean up mock chroot after building the package
+  -m MOCK_CONFIG, --mock-config MOCK_CONFIG
+                        Value to pass with Mock's -r option. Defaults to
+                        "clear", meaning that Mock will use
+                        /etc/mock/clear.cfg.
 
 
 Requirements
@@ -67,27 +96,26 @@ Requirements
 
 In order to run correctly, ``autospec`` requires the following components:
 
- * python3
+* python3
- * correctly configured mock
+* correctly configured mock
 
-If ``autospec`` is not configured to use a license server, then you will
+If ``autospec`` is not configured to use a license server, then it will use the
-need a ``common/licenses`` file -  which should be an up to date list of
+``autospec/license_hashes`` file -  which is a list of licenses to facilitate
-licenses to facilitate automatic license detection during the scan of a
+automatic license detection during the scan of a tarball. For correctness,
-tarball. For correctness, license names should be in the SPDX identifier
+license names should be in the SPDX identifier format. Each line in the file
-format. Each line in the file constitutes a license definition, for example::
+constitutes a license definition, for example::
 
-    750b9d9cc986bfc80b47c9672c48ca615cac0c87 | BSD-3-Clause
+    750b9d9cc986bfc80b47c9672c48ca615cac0c87, BSD-3-Clause
-    175e59be229a5bedc6be93e958a970385bb04a62 | Apache-2.0
+    175e59be229a5bedc6be93e958a970385bb04a62, Apache-2.0
-    794a893e510ca5c15c9c97a609ce47b0df74fc1a | BSD-2-Clause
+    794a893e510ca5c15c9c97a609ce47b0df74fc1a, BSD-2-Clause
 
 
 Control files
 ==============
-
+It is possible to influence precisely how autospec will behave in order to gain
-It is possible to influence precisely how autospec will behave in order to
+fine control over the build itself. These files may be used to alter the default
-gain fine control over the build itself. These files may be used to alter
+behaviour of the configure routine, to blacklist build dependencies from being
-the default behaviour of the configure routine, to blacklist build dependencies
+automatically added, and such.
-from being automatically added, and such.
 
 These files are expected to live in same directory that the resulting ``.spec``
 will live.
@@ -95,215 +123,491 @@ will live.
 Common files
 ------------
 
-**release**
+release
+  This file contains the current release number that will be used in the
+  ``.spec``. This is also bumped and generated on existing and new packages,
+  respectively. This results in less manual work via automatic management.
 
-    This file contains the current release number that will be used in the
+$package.license
-    ``.spec``. This is also bumped and generated on existing and new packages,
+  In certain cases, the package license may not be automatically discovered.  In
-    respectively. This results in less manual work via automatic management.
+  this instance, ``autospec`` will exit with an error. Update this file to
-
+  contain the valid SPDX identifier for any license(s) for the package,
-**$package.license**
+  replacing ``$package`` in the filename with the actual package name.
-
-    In certain cases, the package license may not be automatically discovered.
-    In this instance, ``autospec`` will exit with an error. Update this file
-    to contain the valid SPDX identifier for any license(s) for the package,
-    replacing ``$package`` in the filename with the actual package name.
 
 Controlling dependencies
 -------------------------
 
-**buildreq_add**
+buildreq_add
+  Each line in the file provides the name of a package to add as a build
+  dependency to the ``.spec``.
 
-    Each line in the file provides the name of a package to add
+pkgconfig_add
-    as a build dependency to the ``.spec``.
+  Each line in the file is assumed to be a pkgconfig() build dependency.  Add
+  the pkg-config names here, as ``autospec`` will automatically transform the
+  names into their ``pkgconfig($name)`` style when generating the ``.spec``.
 
-**pkgconfig_add**
+requires_add
+  Each line in the file provides the name of a package to add as a runtime
+  dependency to the ``.spec``.
 
-    Each line in the file is assumed to be a pkgconfig() build dependency.
+${custom}_requires_add
-    Add the pkg-config names here, as ``autospec`` will automatically transform
+  Same as "requires_add" above, but instead of the Requires being placed on the
-    the names into their ``pkgconfig($name)`` style when generating the ``.spec``.
+  ``main`` subpackage, they will be placed on the ``-${custom}`` subpackage.
 
-**buildreq_ban**
+provides_add
+  Each line in the file provides the name of a identifier to add as a Provides
+  to the ``.spec``.
 
-    Each line in the file is a build dependency that under no circumstance
+${custom}_provides_add
-    should be automatically added to the build dependencies. This is useful
+  Same as "provides_add" above, but instead of the Provides being placed on the
-    to block automatic configuration routines adding undesired functionality,
+  ``main`` subpackage, they will be placed on the ``-${custom}`` subpackage.
-    or to omit any automatically discovered dependencies during tarball scanning.
 
-**pkgconfig_ban**
+buildreq_ban
+  Each line in the file is a build dependency that under no circumstance should
+  be automatically added to the build dependencies. This is useful to block
+  automatic configuration routines adding undesired functionality, or to omit
+  any automatically discovered dependencies during tarball scanning.
 
-    Each line in this file is a pkgconfig() build dependency that should not
+pkgconfig_ban
-    be added automatically to the build, much the same as ``buildreq_ban``.
+  Each line in this file is a pkgconfig() build dependency that should not be
-    As with ``pkgconfig_add``, these names are automatically transformed by
+  added automatically to the build, much the same as ``buildreq_ban``.  As with
-    ``autospec`` into their correct ``pkgconfig($name))`` style.
+  ``pkgconfig_add``, these names are automatically transformed by ``autospec``
+  into their correct ``pkgconfig($name))`` style.
 
+requires_ban
+  Each line in the file is a runtime dependency that under no circumstance
+  should be automatically added to the runtime dependencies. This is useful to
+  block automatic configuration routines adding undesired functionality, or to
+  omit any automatically discovered dependencies during tarball scanning.
+
+${custom}_requires_ban
+  Same as "requires_ban" above, but instead of the Requires being removed from
+  the ``main`` subpackage, they will be removed from the ``-${custom}``
+  subpackage.
+
+provides_ban
+  Each line in the file is an identifier that under no circumstance should be
+  automatically added as a Provides. This is useful to block automatic
+  configuration routines adding undesired identifiers.
+
+${custom}_provides_ban
+  Same as "provides_ban" above, but instead of the Provides being removed from
+  the ``main`` subpackage, they will be removed from the ``-${custom}``
+  subpackage.
+
+.. note::
+
+  Run time requirements are not assumed to be build time requirement
+  If a package has the same build and run time requirement it must be added
+  to both buildreq_add and requires_add.
 
 Controlling the build process
 ------------------------------
 
-**configure**
+extra_sources
+  This file contains a list of extra files to be added to the ``.spec`` and
+  optionally installed as well. Each non-blank and non-comment line should start
+  with the file name as found in the Git directory, followed by arguments to be
+  passed to the /usr/bin/install(3) command, with at least one argument starting
+  with a slash, denoting the destination directory (there's no need for
+  ``%{buildroot}``). If the install arguments are missing, Autospec will not
+  generate an installation command and the package should specify how to install
+  in the install_append file (see below).
 
-    This file contains configuration flags to pass to the ``%configure``
+configure
-    macro for autotools based tarballs. As an example, adding ``--disable-static``
+  This file contains configuration flags to pass to the ``%configure`` macro for
-    to ``./configure`` for an autootools based tarball would result in
+  autotools based tarballs. As an example, adding ``--disable-static`` to
-    ``%configure --disable-static`` being emitted in the ``.spec``.
+  ``./configure`` for an autootools based tarball would result in ``%configure
+  --disable-static`` being emitted in the ``.spec``.
 
-**cmake_args**
+configure_openmpi
+  This file contains configuration flags to pass to the ``%configure`` macro for
+  autotools based tarballs to configure openmpi builds.
 
-    This file contains arguments that should be passed to the ``%cmake``
+configure32, configure64, configure_avx2, configure_avx512
-    macro for CMake based tarballs. As an example, adding ``-DUSE_LIB64=ON`` to
+  These files are appended to the ``%configure'' macro after the
-    ``./cmake_args`` would result in ``%cmake -DUSE_LIB64=ON`` being emitted
+  contents of the ``configure'' file above. They are used for 32-bit,
-    in the ``.spec``.
+  regular 64-bit, AVX2 and AVX512 builds, respectively.
 
-**broken_parallel_build**
+cmake_args
+  This file contains arguments that should be passed to the ``%cmake`` macro for
+  CMake based tarballs. As an example, adding ``-DUSE_LIB64=ON`` to
+  ``./cmake_args`` would result in ``%cmake -DUSE_LIB64=ON`` being emitted in
+  the ``.spec``.
 
-    If this file exists, then parallelisation will be disabled in the build.
+cmake_args_openmpi
-    This usually means that ``%{?_smp_mflags}`` will not be passed to ``make``
+  This file contains arguments that should be passed to the ``%cmake`` macro for
+  CMake based tarballs for openmpi builds.
 
-**make_args**
+make_command
+  The contents of this file will be used instead of the ``make``
+  command, i.e. use this if ``make`` should be replace with another build tool
+  like ``ninja``.
 
-    The contents of this file are appended to the ``make`` invocation. This
+make_args
-    may be useful for passing arguments to ``make``, i.e. ``make TOOLDIR=/usr``
+  The contents of this file are appended to the ``make`` invocation. This may be
+  useful for passing arguments to ``make``, i.e. ``make TOOLDIR=/usr``
 
-**make_install_args**
+make32_args
+  The contents of this file are appended to the ``make`` invocation of the 32bit
+  build. It is appended after the make_args content so 32bit specific overrides
+  can be added.
 
-    Much like ``make_args``, this will pass arguments to the ``make install``
+make_install_args
-    macro in the ``.spec``
+  Much like ``make_args``, this will pass arguments to the ``make install``
+  macro in the ``.spec``
 
-**install_macro**
+make32_install_args
+  Much like ``make32_args``, this will pass arguments to the ``make install``
+  macro in the ``.spec`` for the 32bit build. Again it is appended after
+  make_install_args so 32bit specific overrides can be added.
 
-    The contents of this file be used instead of the automatically detected
+prep_prepend
-    ``install`` routine, i.e. use this if ``%make_install`` is insufficient.
+  Additional actions that should take place directly after ``%prep``
+  and before the ``%setup`` macro.  This will be placed in the
+  resulting ``.spec``, and is used for situations where fine-grained
+  control is required.
 
-**subdir**
+build_prepend
+  Additional actions that should take place after ``%build`` and before
+  the ``%configure`` macro or equivalent (``%cmake``, etc.). If autospec
+  is creating AVX2, AVX-512 or 32-bit, these actions will be repeated for
+  each of those builds, This will be placed in the resulting ``.spec``,
+  and is used for situations where fine-grained control is required.
 
-    Not all packages have their ``Makefile``'s available in the root of the tarball.
+build_prepend_once
-    An example of this may be cross-platform projects that split Makefile's into
+  Additional actions that should take place directly after ``%build``
-    the ``unix`` subdirectory. Set the name in this file and the ``.spec`` will
+  and before the ``%configure`` macro or equivalent (``%cmake``, etc.).
-    emit the correct ``pushd`` and ``popd`` lines to utilise these directories
+  If autospec is creating AVX2, AVX-512 or 32-bit, these action will
-    for each step in the build.
+  not be repeated for each of those builds, This will be placed in the
+  resulting ``.spec``, and is used for situations where fine-grained
+  control is required.
 
-**build_pattern**
+make_prepend
+  Additional actions that should take place directly after the
+  configuring step and before the ``%make`` macro or equivalent. If
+  autospec is creating AVX2, AVX-512 or 32-bit, these actions will be
+  repeated for each of those builds, before their respective make
+  steps. This will be placed in the resulting ``.spec``, and is used
+  for situations where fine-grained control is required.
 
-    In certain situations, the automatically detected build pattern may not
+install_prepend
-    work for the given package. This one line file allows you to override the
+  Additional actions that should take place directly after
-    build pattern that ``autospec`` will use. The supported build_pattern types are:
+  ``%install`` but before the ``%make_install`` macro (or equivalent).
+  This will be placed in the resulting ``.spec``, and is used for
+  situations where fine-grained control is required.
 
-        - configure: Traditional ``%configure`` autotools route
+install_append
-        - configure_ac: Like ``configure, but performs ``%reconfigure`` to regenerate ``./configure``
+  Additional actions that should take place at the very end of the
-        - autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh`` instead of ``%reconfigure``
+  ``%install`` section. This will be placed in the resulting ``.spec``,
-        - distutils: Only build the Pythonic package with Python 2
+  and is used for situations where fine-grained control is required.
-        - distutils23: Build the Pythonic package using both Python 2 and Python 3
 
-**series**
+install_macro
+  The contents of this file will be used instead of the automatically detected
+  ``install`` routine, i.e. use this if ``%make_install`` is insufficient.
 
-    This file contains a list of patches to apply during the build, using the ``%patch``
+subdir
-    macro. As such it is affected by ``-p1`` style modifiers.
+  Not all packages have their ``Makefile``'s available in the root of the
+  tarball.  An example of this may be cross-platform projects that split
+  Makefile's into the ``unix`` subdirectory. Set the name in this file and the
+  ``.spec`` will emit the correct ``pushd`` and ``popd`` lines to utilise these
+  directories for each step in the build.
+
+cmake_srcdir
+ The contents of this file are a path to the source directory in which to run
+ cmake for non-standard packages. This path is relative to the clr-build
+ subdirectory, which is created directly below the source package's root.
+
+build_pattern
+  In certain situations, the automatically detected build pattern may not work
+  for the given package. This one line file allows you to override the build
+  pattern that ``autospec`` will use. The supported build_pattern types are:
+
+  * R: R language package
+  * cpan: perl language package
+  * configure: Traditional ``%configure`` autotools route
+  * configure_ac: Like ``configure``, but performs ``%reconfigure`` to
+    regenerate ``./configure``
+  * autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh``
+    instead of ``%reconfigure``
+  * cmake: Traditional builds using CMake
+  * qmake: qmake (Qt5) projects
+  * make: Run ``make`` followed by ``make install``, skipping configure. Note
+    that this is the fallback build pattern in case no other build patterns are
+    autodetected
+  * distutils3: Only build the Pythonic package with Python 3
+  * cargo: Run ```cargo``` to build and install content put in ~/.cargo/bin
+  * pyproject: Build the Pythonic package using the PEP 516 method
+  * meson: Build package with Meson/Ninja
+  * \[WIP\] scons: Build package with Scons
+
+series
+  This file contains a list of patches to apply during the build, using the
+  ``%patch`` macro. As such it is affected by ``-p1`` style modifiers.
+  Arguments to patch can be added after the patch filename.  For example:
+
+  ```
+  0001-my-awesome-patch.patch -d some/subdir -p1
+  ```
+
+pypi_overrides
+  This file contains a list of modules to remove version tests on during the
+  build. It also modifies the requirement of the requires.txt egg if it
+  exists in the package. For example:
+
+  ```
+  colorama
+  ```
+  in the file will cause this modification:
+  ```
+  -    'colorama>=0.2.5,<0.4.4',
+  +    'colorama',
+  ```
+
+service_restart
+  Each line in the file specifies the full path to a systemd unit file
+  installed by this package that should be restarted by clr-service-restart_.
+
+.. _clr-service-restart: https://github.com/clearlinux/clr-service-restart
 
 Controlling files and subpackages
 ---------------------------------
 
-**excludes**
+excludes
+  This file is used to generate ``%exclude`` lines in the ``.spec``. This is
+  useful for omitting files from being included in the resulting package.  Each
+  line in the file should be a full path name.
 
-    This file is used to generate ``%exclude`` lines in the ``.spec``. This
+extras
-    is useful for omitting files from being included in the resulting package.
+  Each line in the file should be a full path within the resulting package, that
-    Each line in the file should be a full path name.
+  you wish to be placed into an automatic ``-extras`` subpackage. This allows
+  one to keep the main package slim and split out optional functionality or
+  files.
 
-**keepstatic**
+  Files paths can contain a single '*' per directory such that
+  a line of ``/foo*/bar*`` is allowed but ``/foo*bar*`` is not.
 
-    If this file exists, then ``%define keepstatic 1`` is emitted in the ``.spec``.
+dev_extras
-    As a result, any static archive (``.a``) files will not be removed by rpmbuild.
+  Same as "extras" above, but instead of the files being placed in an
+  ``-extras`` subpackage, they will be placed in the ``-dev`` one. Use this
+  functionality to place files used only for development against this
+  software that Autospec does not automatically detect.
 
-**extras**
+tests_extras
+  Same as "extras" above, but instead of the files being placed in an
+  ``-extras`` subpackage, they will be placed in the ``-tests`` one. Use this
+  functionality to place files used only for testing against this
+  software that Autospec does not automatically detect.
 
-    Each line in the file should be a full path within the resulting package,
+${custom}_extras
-    that you wish to be placed into an automatic ``-extras`` subpackage. This
+  Same as "extras" above, but instead of the files being placed in an
-    allows one to keep the main package slim and split out optional functionality
+  ``-extras`` subpackage, they will be placed in the ``extras-${custom}``
-    or files.
+  subpackage.
 
-**setuid**
+${custom}_extras_requires
+  Each line contains a subpackage names of other subpackages in the package.
+  This is used when the ``extras-${custom}`` subpackage has a runtime
+  requirement on a sibling subpackage.
 
-    Each line in this file should contain the full path to a binary in the resulting
+  An example of the ``${custom}_extras`` and ``${custom}_extras_requires``
-    build that should have the ``setuid`` attribute set with the ``%attr`` macro.
+  being used together with::
 
-**attrs**
+    /usr/bin/foo
 
-    Each line in this file should be a full ``%attr`` macro line that will be included
+  in foo_extras and::
-    in the ``.spec`` to have fine-grained control over the permissions and ownership
+
-    of files in the package.
+    data
+
+  in foo_extras_requires will produce a spec file package
+  section for example-foo-extras with the following content::
+
+    %package extras-foo
+    Summary: extras-foo components for the example package.
+    Group: Default
+    Requires: example-data = %{version}-%{release}
+
+    %description extras-foo
+    extras-foo components for the example package.
+
+setuid
+  Each line in this file should contain the full path to a binary in the
+  resulting build that should have the ``setuid`` attribute set with the
+  ``%attr`` macro.
+
+attrs
+  Each line in this file should specify mode, user, group and filename
+  (space separated) which is translated into a full ``%attr`` macro
+  line that will be included in the ``.spec`` to have fine-grained control
+  over the permissions and ownership of files in the package.
+
+  An example of a ``attrs`` file would contain::
+
+    4755 root messagebus /usr/libexec/dbus-daemon-launch-helper
+
+  which would translate to the following line in the resulting ``.spec`` file::
+
+    %attr(4755,root,messagebus) /usr/libexec/dbus-daemon-launch-helper
 
 
 Controlling test suites
 -----------------------
-
 By default, ``autospec`` will attempt to detect potential test suites that
 can be run in the ``%check`` portion of the ``.spec``.
 
-**skip_test_suite**
+make_check_command
+  Override or set the command to use in the ``%check`` portion of the ``.spec``.
+  This may be useful when a package uses a custom test suite, or requires
+  additional work/parameters, to work correctly.
 
-    If this file exists, ``autospec`` will not emit any ``%check`` functionality.
+Controlling miscellaneous spec metadata
+---------------------------------------
 
-**unit_tests_must_pass**
+description
+  Provides content for the %description section, overriding the content
+  autospec autodetects. This is useful if autospec cannot find proper content
+  for the description, if one wants to customize the content for better
+  presentation, etc.
 
-    This file is automatically created upon successful completion of a package build.
+summary
-    This allows one to identify regressions in test failures when updating or
-    altering a package.
-    ``autospec`` will fail a package that does not pass it's test suite if this file
-    exists.
 
-**make_check_command**
+  Provides the main Summary: value of the package, overriding any automatically
+  found values. Only the first line is used.
 
-    Override or set the command to use in the ``%check`` portion of the ``.spec``.
+pypi.json
-    This may be useful when a package uses a custom test suite, or requires
+  Provides an alternative to reading the pypi api url for package metadata.
-    additional work/parameters, to work correctly.
+  provides, requires, summary, description and license information could be
-
+  sourced from this file (see https://wiki.python.org/moin/PyPIJSON) for more
-**allow_test_failures**
+  details on the structure.
-
-    The existence of this file will allow test failures, and will still emit
-    the ``%check`` code in a way that allows the build to continue.
 
 
-Controlling flags and optimisation
+Controlling flags and optimization
------------------------------------
+----------------------------------
+Further control of the build can be achieved through the use of the
+``options.conf`` file. If this file does not exist it is created by autospec
+with default values. If certain deprecated configuration files exists autospec
+will use the value indicated by those files and remove them.
 
-**asneeded**
+The options that can be set in ``options.conf`` are as follows:
 
-    If this file exists, the ``.spec`` will disable the LD_AS_NEEDED variable.
+asneeded
-    Supporting binutils (such as found in Clear Linux Project for Intel Architecture)
+  If this is option set, the ``.spec`` will disable the LD_AS_NEEDED variable.
-    will then revert to their normal behaviour, instead of enforcing ``-Wl,-as-needed``
+  Supporting binutils (such as found in Clear Linux Project for Intel
-    in the most correct sense.
+  Architecture) will then revert to their normal behaviour, instead of enforcing
+  ``-Wl,-as-needed`` in the most correct sense.
 
-**optimize_size**
+optimize_size
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build the
+  package optimized for *size*, and not for *speed*. Use this when size is more
+  critical than performance.
 
-    If this file exists, the ``CFLAGS/LDFLAGS`` will be extended to build
+funroll-loops
-    the package optimised for *size*, and not for *speed*. Use this when
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build the
-    size is more critical than performance.
+  package optimized for *speed*. In short this where speed is of paramount
+  importance, and will use ``-03`` by default.
 
-**funroll-loops**
+insecure_build
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be **replaced**, using the
+  smallest ``-02`` based generic flags possible. This is useful for operating
+  systems employing heavy optimizations or full RELRO by default.
 
-    If this file exists, the ``CFLAGS/LDFLAGS`` will be extended to build
+pgo
-    the package optimised for *speed*. In short this where speed is of
+  If this option is set, the ``CFLAGS/CXXFLAGS`` will be extended to build the
-    paramount importance, and will use ``-03`` by default.
+  package with profile-guided optimization data. It will add ``-O3``,
+  ``-fprofile-use``, ``-fprofile-correction`` and ``-fprofile-dir=pgo``.
 
-**insecure_build**
+use_lto
+  If this option is set, link time optimization is enabled for the build.
 
-    If this file exists, the ``CFLAGS/LDFLAGS`` will be **replaced**, using
+use_avx2
-    the smallest ``-02`` based generic flags possible. This is useful for
+  If this option is set, a second set of libraries, for AVX2, is built.
-    operating systems employing heavy optimisations or full RELRO by default.
 
+use_avx512
+  If this option is set, an additional set of libraries, for AVX512, is built.
+
+openmpi
+  If this option is set, an additional openmpi package is built.
+
+fast-math
+  If this option is set, -ffast-math is passed to the compiler.
+
+broken_c++
+  If this option is set, flags are extended with -std=gnu++98.
+
+allow_test_failures
+  If this option is set it will allow test failures, and will still emit the
+  ``%check`` code in a way that allows the build to continue.
+
+skip_tests
+  If this option is set the test suite will not be run.
+
+no_autostart
+  If this option is set the autostart subpackage (which contains all files
+  matching /usr/lib/systemd/system/\*.target.wants/) will not be required by the
+  base package.
+
+conservative_flags
+  If this option is set autospec will set conservative build flags
+
+broken_parallel_build
+  If this option is set, the parallelization is disabled during build.
+
+use_clang
+  If this option is set autospec will utilize clang. This unsets the
+  funroll-loops optimization if it is set.
+
+keepstatic
+  If this option is set, then ``%define keepstatic 1`` is emitted in the
+  ``.spec``.  As a result, any static archive (``.a``) files will not be removed
+  by rpmbuild.
+
+32bit
+  This option will trigger the creation of 32-bit libraries for a 32-bit build.
+
+nostrip
+  This option will suppress the stripping of the created binaries.
+
+verify_required
+  This option will make package verification required for the build. This option
+  is automatically set if package verification is ever successful, but can be
+  turned off manually.
+
+security_sensitive
+  This options sets flags for security-sensitive builds.
+
+so_to_lib
+  This option causes package ``.so`` files to be added to the ``lib`` subpackage
+  instead of the ``dev`` subpackage.
+
+dev_requires_extras
+  If this option is set, the ``extras`` subpackage is marked as a dependency of
+  the ``dev`` package.
+
+autoupdate
+  This option indicates that the package is trusted enough to be automatically
+  update to its newest available version when set to ``true``. This flag is
+  intended to be used by tools running autospec automatically.
+
+compat
+  This option indicates the package is a library compatibility package and only
+  provides versioned library files.
+
+nodebug
+  If this option is set, ``debuginfo`` is not created for this package.
 
 Name and version resolution
 ===========================
 
-``autospec`` will attempt to use a number of patterns to determine the name
+``autospec`` will attempt to use a number of patterns to determine the name and
-and version of the package by examining the URL. For most tarballs this is
+version of the package by examining the URL. For most tarballs this is simple,
-simple, if they are of the format ``$name-$version.tar.$compression``.
+if they are of the format ``$name-$version.tar.$compression``.
 
-For websites such as ``bitbucket`` or ``GitHub``, using ``get$`` and ``v$.tar.*``
+For websites such as ``bitbucket`` or ``GitHub``, using ``get$`` and
-style links, the project name itself is used from the URL and the version is
+``v$.tar.*`` style links, the project name itself is used from the URL and the
-determined by stripping down the tag.
+version is determined by stripping down the tag.
 
-CPAN Perl packages, R packages, and rubygems.org rubygems are automatically
+CPAN Perl packages, pypi ecosystem packages and R packages are automatically
-prefixed with their language name: ``perl-``, ``R-`` and ``rubygem-`` respectively.
+prefixed with their respective names: ``perl-``, ``pypi-`` and ``R-``
+respectively.
 
 When these automated detections are not desirable, it is possible to override
 these with the ``--name`` flag when invoking ``autospec``
@@ -318,14 +622,14 @@ talks HTTP.
 
 This URL should accept ``POST`` requests with the following keys:
 
-**hash**
+hash
-    Contains the SHA-1 hash of the potential license file being checked.
+  Contains the SHA-1 hash of the potential license file being checked.
 
-**package**
+package
-    The name of the package being examined
+  The name of the package being examined
 
-**text**
+text
-    The contents of the potential license file
+  The contents of the potential license file
 
 Implementations return a *plain text* response with the SPDX identifier
 of the license, if known. An empty response is assumed to mean that this
@@ -333,4 +637,13 @@ license is unknown, in which case ``autospec`` will emit the ``license_show``
 URL. The implementation should show the now-stored license file via a
 web page, and enable a human to make a decision on the license. This is
 then stored internally, allowing future requests to automatically know
-the license type when this hash is encounted again. 
+the license type when this hash is encountered again.
+
+
+Integration of systemd unit files
+=================================
+``autospec`` can add most systemd template file types by having a file in the
+filename.extension in the build directory. Supported extensions are:
+``mount, service, socket, target, timer, path and tmpfiles``. The files will
+be added as Source# entries and be installed to their appropriate system
+location.

autospec/__init__.py

@@ -1,6 +1,5 @@
-import sys
+"""Autospec, an automated specfile generation utility."""
-import os
 
-__all__ = ["buildreq", "build", "buildpattern", "config", "docs","files",
+__all__ = ["abireport", "buildreq", "build", "config", "files",
-           "git","lang", "license", "patches", "specdescription", "tarball",
+           "git", "lang", "license", "patches", "specdescription",
-           "util", "commitmessage", "test", "patches"]
+           "tarball", "util", "commitmessage", "test", "patches"]

autospec/abireport.py

@@ -0,0 +1,336 @@
+#!/bin/true
+#
+# abireport.py - part of autospec
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Generate a symbols file from all shared libraries encountered, enabling
+# a consistent ABI report for every build. We ensure that everything is
+# appropriately sorted, in that a diff only occurs when the shared libraries
+# in the package themselves actually change too.
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+import util
+
+valid_dirs = ["/usr/lib", "/usr/lib64"]
+
+# For determining .so's
+reg = re.compile(r".* ELF (64|32)\-bit LSB shared object,")
+
+# All dynamic binaries
+valid_dyn = re.compile(r".* ELF (64|32)\-bit LSB (shared object|executable),")
+
+# shared-lib matcher
+shared_lib = re.compile(r".*Shared library: \[(.*)\].*")
+
+wanted_symbol_types = ["A", "T"]
+
+ignored_symbols = [
+    "__bss_start",
+    "_edata",
+    "_end",
+    "_fini",
+    "_init",
+]
+
+
+def get_output(cmd):
+    """Return output from subprocess.getoutput."""
+    try:
+        o = subprocess.getoutput(cmd)
+        return o
+    except Exception as e:
+        print("Error: %s" % e)
+
+
+def get_soname(path):
+    """Use objdump to find the SONAME of a file."""
+    cmd = "objdump -p \"{}\"|grep SONAME".format(path)
+    try:
+        line = get_output(cmd)
+        if "SONAME" not in line:
+            return None
+        line = line.strip()
+        spl = line.split()[1]
+        return spl
+    except Exception:
+        return None
+
+
+def get_shared_dependencies(path):
+    """Return the shared dependencies for a given path."""
+    ret = set()
+    cmd = "readelf -d {}".format(path)
+
+    for line in get_output(cmd).split("\n"):
+        line = line.strip()
+        shared = shared_lib.match(line)
+        if shared is None:
+            continue
+        ret.add(shared.group(1))
+
+    return ret
+
+
+def get_all_dependencies(path):
+    """Determine all dependencies in the given path."""
+    deps = set()
+
+    sonames = set()
+    examine = set()
+
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            fpath = os.path.join(root, file)
+            if not is_dynamic_binary(fpath):
+                continue
+            # Encountered a valid dynamic linked object
+            if is_file_valid(fpath):
+                # We must account for *all* internal symbols due to rpaths and
+                # overriding of LD_LIBRARY_PATH
+                soname = get_soname(fpath)
+                if soname is not None:
+                    sonames.add(soname)
+            if is_dynamic_binary(fpath):
+                examine.add(fpath)
+
+    for path in examine:
+        current_deps = get_shared_dependencies(path)
+        # Ensure we don't add a dependency on an internally provided symbol
+        deps.update(set(filter(lambda s: s not in sonames, current_deps)))
+
+    return deps
+
+
+def get_file_magic(path):
+    """Return the 'magic' for a given path."""
+    cmd = "file \"{}\"".format(path)
+    try:
+        line = get_output(cmd).split("\n")[0]
+        return line.strip()
+    except Exception:
+        return None
+
+
+def is_dynamic_binary(path):
+    """Determine if a given path is a dynamic binary."""
+    if not os.path.exists(path) or not os.path.isfile(path):
+        return False
+    mg = get_file_magic(path)
+    if not mg:
+        return False
+    if valid_dyn.match(mg):
+        return True
+    return False
+
+
+def is_file_valid(path):
+    """Validate file is an SO."""
+    if not os.path.exists(path) or os.path.islink(path):
+        return False
+    mg = get_file_magic(path)
+    if not mg:
+        return False
+    if reg.match(mg):
+        return True
+    return False
+
+
+def dump_symbols(path):
+    """Get symbols from a file."""
+    cmd = "nm --defined-only -g --dynamic \"{}\"".format(path)
+    lines = None
+
+    ret = set()
+
+    try:
+        lines = get_output(cmd)
+    except Exception as e:
+        print("Fatal error inspecting {}: {}".format(path, e))
+        sys.exit(1)
+    for line in lines.split("\n"):
+        line = line.strip()
+
+        spl = line.split()
+        if len(spl) != 3:
+            continue
+        sym_type = spl[1]
+        sym_id = spl[2]
+
+        if sym_type not in wanted_symbol_types:
+            continue
+        if sym_id in ignored_symbols:
+            continue
+        ret.add(sym_id)
+    return ret
+
+
+def purge_tree(tree):
+    """Run rm -fr."""
+    if not os.path.exists(tree):
+        return
+    try:
+        shutil.rmtree(tree)
+    except Exception as e:
+        util.print_fatal("Cannot remove tree: {}".format(e))
+        sys.exit(1)
+
+
+def truncate_file(path):
+    """Zero file content."""
+    if not os.path.exists(path):
+        return
+    with util.open_auto(path, "r+") as trunc:
+        trunc.truncate()
+
+
+def examine_abi(download_path, name):
+    """Proxy the ABI reporting to the right function."""
+    download_path = os.path.abspath(download_path)
+    results_dir = os.path.abspath(os.path.join(download_path, "results"))
+
+    if not os.path.exists(results_dir):
+        util.print_fatal("Results directory does not exist, aborting")
+        sys.exit(1)
+
+    if util.binary_in_path("abireport"):
+        examine_abi_host(download_path, results_dir, name)
+    else:
+        util.print_warning("abireport is not installed. Using slow scanning")
+        examine_abi_fallback(download_path, results_dir, name)
+
+
+def examine_abi_host(download_path, results_dir, name):
+    """Make use of the hostside abireport tool."""
+    rpms = set()
+    for item in os.listdir(results_dir):
+        namelen = len(name)
+        if item.find("-extras-", namelen) >= namelen:
+            continue
+        if item.endswith(".rpm") and not item.endswith(".src.rpm"):
+            rpms.add("{}/{}".format(results_dir, item))
+
+    if len(rpms) == 0:
+        util.print_fatal("No usable rpms found, aborting")
+        sys.exit(1)
+
+    try:
+        util.call("abireport scan-packages {}".format(" ".join(rpms)),
+                  cwd=download_path)
+    except Exception as e:
+        util.print_fatal("Error invoking abireport: {}".format(e))
+
+
+def examine_abi_fallback(download_path, results_dir, name):
+    """Missing abireport so fallback to internal scanning."""
+    old_dir = os.getcwd()
+
+    rpms = set()
+    for item in os.listdir(results_dir):
+        namelen = len(name)
+        if item.find("-extras-", namelen) >= namelen:
+            continue
+        if item.endswith(".rpm") and not item.endswith(".src.rpm"):
+            rpms.add(os.path.basename(item))
+
+    if len(rpms) == 0:
+        util.print_fatal("No usable rpms found, aborting")
+        sys.exit(1)
+
+    extract_dir = os.path.abspath(os.path.join(download_path, "__extraction"))
+    purge_tree(extract_dir)
+
+    try:
+        os.makedirs(extract_dir)
+    except Exception as e:
+        util.print_fatal("Cannot create extraction tree: {}".format(e))
+        sys.exit(1)
+
+    os.chdir(extract_dir)
+
+    # Extract all those rpms to our current directory
+    try:
+        for rpm in rpms:
+            cmd = "rpm2cpio \"{}\" | cpio -imd 2>/dev/null".format(os.path.join(results_dir, rpm))
+            subprocess.check_call(cmd, shell=True)
+    except Exception as e:
+        util.print_fatal("Error extracting RPMS: {}".format(e))
+
+    os.chdir(download_path)
+    collected_files = set()
+
+    # Places we expect to find shared libraries
+    for check_path in valid_dirs:
+        if check_path[0] == '/':
+            check_path = check_path[1:]
+
+        dirn = os.path.join(extract_dir, check_path)
+        if not os.path.isdir(dirn):
+            continue
+
+        for file in os.listdir(dirn):
+            f = os.path.basename(file)
+
+            clean_path = os.path.abspath(os.path.join(dirn, f))
+            if not is_file_valid(clean_path):
+                continue
+            collected_files.add(clean_path)
+
+    abi_report = dict()
+
+    # Now examine these libraries
+    for library in sorted(collected_files):
+        soname = get_soname(library)
+        if not soname:
+            warn = "Failed to determine soname of: {}".format(library)
+            util.print_warning(warn)
+            soname = os.path.basename(library)
+        symbols = dump_symbols(library)
+        if symbols and len(symbols) > 0:
+            if soname not in abi_report:
+                abi_report[soname] = set()
+            abi_report[soname].update(symbols)
+
+    report_file = os.path.join(download_path, "symbols")
+
+    if len(abi_report) > 0:
+        # Finally, write the report
+        report = util.open_auto(report_file, "w")
+        for soname in sorted(abi_report.keys()):
+            for symbol in sorted(abi_report[soname]):
+                report.write("{}:{}\n".format(soname, symbol))
+
+        report.close()
+    else:
+        truncate_file(report_file)
+
+    # Write the library report
+    lib_deps = get_all_dependencies(extract_dir)
+    report_file = os.path.join(download_path, "used_libs")
+    if len(lib_deps) > 0:
+        report = util.open_auto(report_file, "w")
+        for soname in sorted(lib_deps):
+            report.write("{}\n".format(soname))
+        report.close()
+    else:
+        truncate_file(report_file)
+
+    os.chdir(old_dir)
+    purge_tree(extract_dir)

autospec/autospec.py

@@ -17,115 +17,113 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
-import sys
+import configparser
 import os
 import re
-import types
+import sys
+import tempfile
 
 import build
-import buildpattern
 import buildreq
+import check
+import commitmessage
 import config
 import files
 import git
-import lang
 import license
-import docs
+import pkg_integrity
-import patches
+import pkg_scan
 import specdescription
+import specfiles
 import tarball
-import test
+from abireport import examine_abi
-import commitmessage
+from logcheck import logcheck
-
+from util import binary_in_path, print_fatal, write_out
-from tarball import name
-from util import _file_write
 
 sys.path.append(os.path.dirname(__file__))
 
-def write_sources(file):
+
-    """Append additonal source files.
+def check_requirements(use_git):
-    systemd unit files, gcov and additional source tarballs
+    """Ensure all requirements are satisfied before continuing."""
-    are the currently supported additonal source file types."""
+    required_bins = ["mock", "rpm2cpio", "nm", "objdump", "cpio", "readelf"]
-    source_count = 1
+
-    for source in sorted(buildpattern.sources["unit"] +
+    if use_git:
-                         buildpattern.sources["archive"] +
+        required_bins.append("git")
-                         buildpattern.sources["tmpfile"] +
+
-                         buildpattern.sources["gcov"]):
+    missing = [x for x in required_bins if not binary_in_path(x)]
-        buildpattern.source_index[source] = source_count
+
-        file.write("Source{0}  : {1}\n".format(source_count, source))
+    if missing:
-        source_count += 1
+        print_fatal("Required programs are not installed: {}".format(", ".join(missing)))
+        sys.exit(1)
 
 
-def write_spec(filename):
+def load_specfile(conf, specfile):
-    file = open(filename, "w", encoding="utf-8")
+    """Gather all information from static analysis into Specfile instance."""
-    file.write_strip = types.MethodType(_file_write, file)
+    specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ)
-    file.write("#\n")
+    license.load_specfile(specfile)
-    file.write("# This file is auto-generated. DO NOT EDIT\n")
+    check.load_specfile(specfile)
-    file.write("# Generated by: autospec.py\n")
-    file.write("#\n")
-
-    if config.keepstatic == 1:
-        file.write("%define keepstatic 1\n")
-
-    # first, write the general package header
-    tarball.write_nvr(file)
-    write_sources(file)
-    specdescription.write_summary(file)
-    license.write_license(file)
-
-    files.write_main_subpackage_requires(file)
-    buildreq.write_buildreq(file)
-    patches.write_patch_header(file)
-
-    # then write the main package extra content
-    specdescription.write_description(file)
-    files.write_files_header(file)
-
-    # then write the build instructions
-    buildpattern.write_buildpattern(file)
-
-    # then write the scriplets
-    files.write_scriplets(file)
-
-    # then write the %files
-    files.write_files(file)
-    lang.write_lang_files(file)
-
-    file.close()
 
 
-def add_sources(download_path, archives):
+def read_old_metadata():
-    for file in os.listdir(download_path):
+    """Handle options.conf providing package, url and archives."""
-        if re.search(".*\.(mount|service|socket|target)$", file):
+    if not os.path.exists(os.path.join(os.getcwd(), 'options.conf')):
-            buildpattern.sources["unit"].append(file)
+        return None, None, []
-    buildpattern.sources["unit"].sort()
+
-    #
+    config_f = configparser.ConfigParser(interpolation=None)
-    # systemd-tmpfiles uses the configuration files from
+    config_f.read('options.conf')
-    # /usr/lib/tmpfiles.d/ directories to describe the creation,
+    if "package" not in config_f.sections():
-    # cleaning and removal of volatile and temporary files and
+        return None, None, []
-    # directories which usually reside in directories such as
+
-    # /run or /tmp.
+    archives = config_f["package"].get("archives")
-    #
+    archives = archives.split() if archives else []
-    if os.path.exists(os.path.normpath(build.download_path +
+    print("ARCHIVES {}".format(archives))
-                                       "/{0}.tmpfiles".format(tarball.name))):
+
-        buildpattern.sources["tmpfile"].append(
+    return (config_f["package"].get("name"),
-            "{}.tmpfiles".format(tarball.name))
+            config_f["package"].get("url"),
-    if tarball.gcov_file:
+            archives)
-        buildpattern.sources["gcov"].append(tarball.gcov_file)
+
-    for archive, destination in zip(archives[::2], archives[1::2]):
+
-        buildpattern.sources["archive"].append(archive)
+def save_mock_logs(path, iteration):
-        buildpattern.archive_details[archive + "destination"] = destination
+    """Save Mock build logs to <path>/results/round<iteration>-*.log."""
+    basedir = os.path.join(path, "results")
+    loglist = ["build", "root", "srpm-build", "srpm-root", "mock_srpm", "mock_build"]
+    for log in loglist:
+        src = "{}/{}.log".format(basedir, log)
+        dest = "{}/round{}-{}.log".format(basedir, iteration, log)
+        os.rename(src, dest)
+
+
+def write_prep(conf, workingdir, content):
+    """Write metadata to the local workingdir when --prep-only is used."""
+    if conf.urlban:
+        used_url = re.sub(conf.urlban, "localhost", content.url)
+    else:
+        used_url = content.url
+
+    print()
+    print("Exiting after prep due to --prep-only flag")
+    print()
+    print("Results under ./workingdir")
+    print("Source  (./workingdir/{})".format(content.tarball_prefix))
+    print("Name    (./workingdir/name)    :", content.name)
+    print("Version (./workingdir/version) :", content.version)
+    print("URL     (./workingdir/source0) :", used_url)
+    write_out(os.path.join(workingdir, "name"), content.name)
+    write_out(os.path.join(workingdir, "version"), content.version)
+    write_out(os.path.join(workingdir, "source0"), used_url)
 
 
 def main():
+    """Entry point for autospec."""
     parser = argparse.ArgumentParser()
     parser.add_argument("-g", "--skip-git",
                         action="store_false", dest="git", default=True,
                         help="Don't commit result to git")
-    parser.add_argument("-n", "--name", nargs=1,
+    parser.add_argument("-n", "--name", action="store", dest="name", default="",
-                        action="store", dest="name", default="",
                         help="Override the package name")
-    parser.add_argument("url",
+    parser.add_argument("-v", "--version", action="store", dest="version", default="",
+                        help="Override the package version")
+    parser.add_argument("url", default="", nargs="?",
                         help="tarball URL (e.g."
                              " http://example.com/downloads/mytar.tar.gz)")
     parser.add_argument('-a', "--archives", action="store",
@@ -144,72 +142,187 @@ def main():
                         default="/usr/share/defaults/autospec/autospec.conf",
                         help="Set configuration file to use")
     parser.add_argument("-t", "--target", dest="target", action="store",
-                        default=None,
+                        required=True,
                         help="Target location to create or reuse")
+    parser.add_argument("-i", "--integrity", action="store_true",
+                        default=False,
+                        help="Search for package signature from source URL and "
+                             "attempt to verify package")
+    parser.add_argument("-p", "--prep-only", action="store_true",
+                        default=False,
+                        help="Only perform preparatory work on package")
+    parser.add_argument("--non_interactive", action="store_true",
+                        default=False,
+                        help="Disable interactive mode for package verification")
+    parser.add_argument("-C", "--cleanup", dest="cleanup", action="store_true",
+                        default=False,
+                        help="Clean up mock chroot after building the package")
+    parser.add_argument("-m", "--mock-config", action="store", default="clear",
+                        help="Value to pass with Mock's -r option. Defaults to "
+                             "\"clear\", meaning that Mock will use "
+                             "/etc/mock/clear.cfg.")
+    parser.add_argument("-o", "--mock-opts", action="store", default="",
+                        help="Arbitrary options to pass down to mock when "
+                        "building a package.")
 
     args = parser.parse_args()
-    if len(args.archives) % 2 != 0:
+
+    name, url, archives = read_old_metadata()
+    name = args.name or name
+    url = args.url or url
+    archives = args.archives or archives
+
+    if not args.target:
         parser.error(argparse.ArgumentTypeError(
-                     "-a/--archives requires an even number of arguments"))
+            "The target option is not valid"))
+    else:
+        # target path must exist or be created
+        os.makedirs(args.target, exist_ok=True)
+
+    if not url:
+        parser.error(argparse.ArgumentTypeError(
+            "the url argument or options.conf['package']['url'] is required"))
+
+    if len(archives) % 2 != 0:
+        parser.error(argparse.ArgumentTypeError(
+            "-a/--archives or options.conf['package']['archives'] requires an "
+            "even number of arguments"))
+
+    if args.prep_only:
+        os.makedirs("workingdir", exists_ok=True)
+        package(args, url, name, archives, "./workingdir")
+    else:
+        with tempfile.TemporaryDirectory() as workingdir:
+            package(args, url, name, archives, workingdir)
+
+
+def package(args, url, name, archives, workingdir):
+    """Entry point for building a package with autospec."""
+    conf = config.Config(args.target)
+    check_requirements(args.git)
+    conf.detect_build_from_url(url)
+    package = build.Build()
+
     #
     # First, download the tarball, extract it and then do a set
     # of static analysis on the content of the tarball.
     #
-    build.setup_patterns()
+    filemanager = files.FileManager(conf, package)
+    content = tarball.Content(url, name, args.version, archives, conf, workingdir)
+    content.process(filemanager)
+    conf.content = content  # hack to avoid recursive dependency on init
+    # Search up one level from here to capture multiple versions
+    _dir = content.path
 
-    tarball.download_tarball(args.url, args.name, args.archives, args.target)
+    conf.setup_patterns()
-    dir = tarball.path
+    conf.config_file = args.config
-    if args.license_only:
+    requirements = buildreq.Requirements(content.url)
-        try:
+    requirements.set_build_req(conf)
-            with open(os.path.join(build.download_path,
+    conf.parse_config_files(args.bump, filemanager, content.version, requirements)
-                      tarball.name + ".license"), "r") as dotlic:
+    conf.setup_patterns(conf.failed_pattern_dir)
-                for word in dotlic.read().split():
+    conf.parse_existing_spec(content.name)
-                    if word.find(":") < 0:
+
-                        license.add_license(word)
+    if args.prep_only:
-        except:
+        write_prep(conf, workingdir, content)
-            pass
-        license.scan_for_licenses(name, dir)
         exit(0)
 
-    config.config_file = args.config
+    if args.license_only:
-    config.parse_config_files(build.download_path, args.bump)
+        try:
-    config.parse_existing_spec(build.download_path, tarball.name)
+            with open(os.path.join(conf.download_path,
+                                   content.name + ".license"), "r") as dotlic:
+                for word in dotlic.read().split():
+                    if ":" not in word:
+                        license.add_license(word)
+        except Exception:
+            pass
+        # Start one directory higher so we scan *all* versions for licenses
+        license.scan_for_licenses(os.path.dirname(_dir), conf, name)
+        exit(0)
 
-    buildreq.scan_for_configure(name, dir, build.download_path)
+    requirements.scan_for_configure(_dir, content.name, conf)
-    specdescription.scan_for_description(name, dir)
+    specdescription.scan_for_description(content.name, _dir, conf.license_translations, conf.license_blacklist)
-    license.scan_for_licenses(name, dir)
+    # Start one directory higher so we scan *all* versions for licenses
-    docs.scan_for_changes(build.download_path, dir)
+    license.scan_for_licenses(os.path.dirname(_dir), conf, content.name)
-    add_sources(build.download_path, args.archives)
+    commitmessage.scan_for_changes(conf.download_path, _dir, conf.transforms)
-    test.scan_for_tests(dir)
+    conf.add_sources(archives, content)
+    check.scan_for_tests(_dir, conf, requirements, content)
 
     #
     # Now, we have enough to write out a specfile, and try to build it.
     # We will then analyze the build result and learn information until the
     # package builds
     #
-    write_spec(build.download_path + "/" + tarball.name + ".spec")
+    specfile = specfiles.Specfile(content.url,
+                                  content.version,
+                                  content.name,
+                                  content.release,
+                                  conf,
+                                  requirements,
+                                  content)
+    filemanager.load_specfile(specfile)
+    load_specfile(conf, specfile)
 
     print("\n")
+
+    if args.integrity:
+        interactive_mode = not args.non_interactive
+        pkg_integrity.check(url, conf, interactive=interactive_mode)
+        pkg_integrity.load_specfile(specfile)
+
+    specfile.write_spec()
     while 1:
-        build.package()
+        package.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, content, args.cleanup)
-        write_spec(build.download_path + "/" + tarball.name + ".spec")
+        filemanager.load_specfile(specfile)
-        files.newfiles_printed = 0
+        specfile.write_spec()
-        if build.round > 20 or build.must_restart == 0:
+        filemanager.newfiles_printed = 0
+        mock_chroot = "/var/lib/mock/clear-{}/root/builddir/build/BUILDROOT/" \
+                      "{}-{}-{}.x86_64".format(package.uniqueext,
+                                               content.name,
+                                               content.version,
+                                               content.release)
+        if filemanager.clean_directories(mock_chroot):
+            # directories added to the blacklist, need to re-run
+            package.must_restart += 1
+
+        if package.round > 20 or (package.must_restart == 0 and package.file_restart == 0):
             break
 
-    test.check_regression(build.download_path)
+        save_mock_logs(conf.download_path, package.round)
 
-    if build.success == 0:
+    if package.success == 0:
-        print("Build failed")
+        conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
-        return
+        print_fatal("Build failed, aborting")
+        sys.exit(1)
+    elif os.path.isfile("README.clear"):
+        try:
+            print("\nREADME.clear CONTENTS")
+            print("*********************")
+            with open("README.clear", "r") as readme_f:
+                print(readme_f.read())
 
-    with open(build.download_path + "/release", "w") as fp:
+            print("*********************\n")
-        fp.write(tarball.release + "\n")
+        except Exception:
+            pass
 
-    commitmessage.guess_commit_message()
+    check.check_regression(conf.download_path, conf.config_opts['skip_tests'], package.round - 1)
+
+    examine_abi(conf.download_path, content.name)
+    if os.path.exists("/var/lib/rpm"):
+        pkg_scan.get_whatrequires(content.name, conf.yum_conf)
+
+    write_out(conf.download_path + "/release", content.release + "\n")
+
+    # record logcheck output
+    logcheck(conf.download_path)
+
+    commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf, content)
+    conf.create_buildreq_cache(content.version, requirements.buildreqs_cache)
 
     if args.git:
-        git.commit_to_git(build.download_path)
+        git.commit_to_git(conf, content.name, package.success)
+    else:
+        print("To commit your changes, git add the relevant files and "
+              "run 'git commit -F commitmsg'")
 
 
 if __name__ == '__main__':

autospec/buildpattern.py

@@ -1,379 +0,0 @@
-#!/bin/true
-#
-# buildpattern.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Deduce and emit the patterns for %build
-#
-
-import config
-import files
-import lang
-import os
-import patches
-import re
-import tarball
-import test
-
-default_pattern = "make"
-pattern_strengh = 0
-
-disable_static = "--disable-static"
-
-extra_make = ""
-extra_cmake = ""
-extra_make_install = ""
-install_macro = "%make_install"
-make_install_append = []
-subdir = ""
-sources = {"unit": [], "gcov": [], "tmpfile": [], "archive": []}
-source_index = {}
-archive_details = {}
-
-
-def write_prep(file, ruby_pattern=False):
-    file.write_strip("%prep")
-    for archive in sources["archive"]:
-        file.write_strip("tar -xf %{{SOURCE{}}}".format(source_index[archive]))
-        file.write_strip("cd ..")
-    if ruby_pattern:
-        file.write_strip("gem unpack %{SOURCE0}")
-        file.write_strip("%setup -q -D -T -n " + tarball.tarball_prefix)
-        file.write_strip("gem spec %{SOURCE0} -l --ruby > " + tarball.name + ".gemspec")
-    else:
-        if default_pattern == 'R':
-            file.write_strip("%setup -q -c -n " + tarball.tarball_prefix)
-        else:
-            file.write_strip("%setup -q -n " + tarball.tarball_prefix)
-    for archive in sources["archive"]:
-        file.write_strip('mv %{{_topdir}}/BUILD/{0}/* ./{1}'
-                         .format(archive_details[archive + "prefix"],
-                                 archive_details[archive + "destination"]))
-    patches.apply_patches(file)
-    file.write_strip("\n")
-
-
-def write_variables(file):
-    flags = []
-    if config.optimize_size:
-        flags.extend(["-Os", "-ffunction-sections"])
-    if config.insecure_build:
-        file.write_strip("export CFLAGS=\"-O2 -g\"\n")
-        file.write_strip("unset LDFLAGS\n")
-    if config.optimize_speed:
-        flags.extend(["-O3", "-flto", "-ffunction-sections", "-fno-semantic-interposition"])
-        file.write_strip("export AR=gcc-ar\n")
-        file.write_strip("export RANLIB=gcc-ranlib\n")
-    if tarball.gcov_file:
-        flags = list(filter(("-flto").__ne__, flags))
-        flags.extend(["-O3", "-fauto-profile=%{{SOURCE{0}}}".format(source_index[sources["gcov"][0]])])
-    if flags:
-        flags = list(set(flags))
-        file.write_strip("export CFLAGS=\"$CFLAGS {0} \"\n".format(" ".join(flags)))
-        file.write_strip("export CXXFLAGS=\"$CXXFLAGS {0} \"\n".format(" ".join(flags)))
-
-
-def write_check(file):
-    if test.tests_config:
-        file.write_strip("%check")
-        file.write_strip("export http_proxy=http://127.0.0.1:9/")
-        file.write_strip("export https_proxy=http://127.0.0.1:9/")
-        file.write_strip("export no_proxy=localhost")
-        file.write_strip(test.tests_config)
-        file.write_strip("\n")
-
-
-def write_make_install(file):
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("%s %s\n" % (install_macro, extra_make_install))
-    if subdir:
-        file.write_strip("popd")
-    lang.write_find_lang(file)
-
-
-def write_configure_pattern(file):
-    if patches.autoreconf:
-        # Patches affecting configure.* or Makefile.*, reconf instead
-        write_configure_ac_pattern(file)
-        return
-    write_prep(file)
-    file.write_strip("%build")
-    if config.asneeded == 0:
-        file.write_strip("unset LD_AS_NEEDED\n")
-    write_variables(file)
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("%configure " + disable_static + " " + config.extra_configure)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    if subdir:
-        file.write_strip("popd")
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_configure_ac_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    if config.asneeded == 0:
-        file.write_strip("unset LD_AS_NEEDED\n")
-    write_variables(file)
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("%reconfigure " + disable_static + " " + config.extra_configure)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    if subdir:
-        file.write_strip("popd")
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_make_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    if subdir:
-        file.write_strip("pushd %s" % subdir)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_autogen_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("%autogen " + disable_static + " " + config.extra_configure)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_distutils_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("python2 setup.py build -b py2 " + config.extra_configure)
-    file.write_strip("\n")
-    if test.tests_config:
-        file.write_strip("%check")
-        # Prevent setuptools from hitting the internet
-        file.write_strip("export http_proxy=http://127.0.0.1:9/")
-        file.write_strip("export https_proxy=http://127.0.0.1:9/")
-        file.write_strip("export no_proxy=localhost,127.0.0.1,0.0.0.0")
-        file.write_strip(test.tests_config)
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("python2 -tt setup.py build -b py2 install --root=%{buildroot}")
-    lang.write_find_lang(file)
-
-
-def write_distutils23_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("python2 setup.py build -b py2 " + config.extra_configure)
-    file.write_strip("python3 setup.py build -b py3 " + config.extra_configure)
-    file.write_strip("\n")
-    if test.tests_config:
-        file.write_strip("%check")
-        # Prevent setuptools from hitting the internet
-        file.write_strip("export http_proxy=http://127.0.0.1:9/")
-        file.write_strip("export https_proxy=http://127.0.0.1:9/")
-        file.write_strip("export no_proxy=localhost,127.0.0.1,0.0.0.0")
-        file.write_strip(test.tests_config)
-
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("python2 -tt setup.py build -b py2 install --root=%{buildroot}")
-    file.write_strip("python3 -tt setup.py build -b py3 install --root=%{buildroot}")
-    lang.write_find_lang(file)
-
-
-def write_R_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    file.write_strip("\n")
-
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("export LANG=C")
-    file.write_strip("export CFLAGS=\"$CFLAGS -O3 -flto -ffunction-sections -fno-semantic-interposition \"\n")
-    file.write_strip("export CXXFLAGS=\"$CXXFLAGS -O3 -flto -ffunction-sections -fno-semantic-interposition \"\n")
-    file.write_strip("export AR=gcc-ar\n")
-    file.write_strip("export RANLIB=gcc-ranlib\n")
-    file.write_strip("export LDFLAGS=\"$LDFLAGS  -Wl,-z -Wl,relro\"\n")
-
-    file.write_strip("mkdir -p %{buildroot}/usr/lib64/R/library")
-    file.write_strip("R CMD INSTALL --install-tests --build  -l %{buildroot}/usr/lib64/R/library " + tarball.rawname)
-    file.write_strip("%{__rm} -rf %{buildroot}%{_datadir}/R/library/R.css")
-    lang.write_find_lang(file)
-    write_check(file)
-
-
-def write_ruby_pattern(file):
-    write_prep(file, ruby_pattern=True)
-    file.write_strip("%build")
-    file.write_strip("gem build " + tarball.name + ".gemspec")
-    file.write_strip("\n")
-
-    file.write_strip("%install")
-    file.write_strip("%global gem_dir $(ruby -e'puts Gem.default_dir')")
-    file.write_strip("gem install -V \\")
-    file.write_strip("  --local \\")
-    file.write_strip("  --force \\")
-    file.write_strip("  --install-dir .%{gem_dir} \\")
-    file.write_strip("  --bindir .%{_bindir} \\")
-    file.write_strip(" " + tarball.tarball_prefix + ".gem")
-    file.write_strip("\n")
-
-    file.write_strip("mkdir -p %{buildroot}%{gem_dir}")
-    file.write_strip("cp -pa .%{gem_dir}/* \\")
-    file.write_strip("        %{buildroot}%{gem_dir}")
-    file.write_strip("\n")
-
-    file.write_strip("if [ -d .%{_bindir} ]; then")
-    file.write_strip("    mkdir -p %{buildroot}%{_bindir}")
-    file.write_strip("    cp -pa .%{_bindir}/* \\")
-    file.write_strip("        %{buildroot}%{_bindir}/")
-    file.write_strip("fi")
-    file.write_strip("\n")
-    lang.write_find_lang(file)
-    write_check(file)
-
-
-def write_cmake_pattern(file):
-    global subdir
-    subdir = "clr-build"
-    write_prep(file)
-    file.write_strip("%build")
-    file.write_strip("mkdir clr-build")
-    file.write_strip("pushd clr-build")
-    file.write_strip("cmake .. -G \"Unix Makefiles\" -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS:BOOL=ON -DLIB_INSTALL_DIR:PATH=%{_libdir} " + extra_cmake)
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("popd")
-    file.write_strip("\n")
-    write_check(file)
-    write_make_install(file)
-
-
-def write_cpan_pattern(file):
-    global subdir
-    write_prep(file)
-    file.write_strip("%build")
-    file.write_strip("if test -f Makefile.PL; then")
-    file.write_strip("%{__perl} Makefile.PL")
-    file.write_strip("make V=1 " + config.parallel_build + extra_make)
-    file.write_strip("else")
-    file.write_strip("%{__perl} Build.PL")
-    file.write_strip("./Build")
-    file.write_strip("fi")
-    file.write_strip("\n")
-    write_check(file)
-    file.write_strip("%install")
-    file.write_strip("rm -rf %{buildroot}")
-    file.write_strip("if test -f Makefile.PL; then")
-    file.write_strip("make pure_install PERL_INSTALL_ROOT=%{buildroot}")
-    file.write_strip("else")
-    file.write_strip("./Build install --installdirs=site --destdir=%{buildroot}")
-    file.write_strip("fi")
-    file.write_strip("find %{buildroot} -type f -name .packlist -exec rm -f {} ';'")
-    file.write_strip("find %{buildroot} -depth -type d -exec rmdir {} 2>/dev/null ';'")
-    file.write_strip("find %{buildroot} -type f -name '*.bs' -empty -exec rm -f {} ';'")
-    file.write_strip("%{_fixperms} %{buildroot}/*")
-    lang.write_find_lang(file)
-
-
-def write_scons_pattern(file):
-    write_prep(file)
-    file.write_strip("%build")
-    write_variables(file)
-    file.write_strip("scons" + config.parallel_build + " " + config.extra_configure)
-    file.write_strip("\n")
-    file.write_strip("%install")
-    file.write_strip("scons install " + extra_make_install)
-
-
-def set_build_pattern(pattern, strength):
-    global default_pattern
-    global pattern_strengh
-    if strength <= pattern_strengh:
-        return 0
-    default_pattern = pattern
-    pattern_strengh = strength
-
-
-def get_systemd_units():
-    """get systemd unit files from the files module"""
-    service_file_section = "config"
-    systemd_service_pattern = r"^/usr/lib/systemd/system/[^/]*\.(mount|service|socket|target)$"
-    systemd_units = []
-
-    if service_file_section not in files.packages:
-        return systemd_units
-
-    for f in files.packages[service_file_section]:
-        if re.search(systemd_service_pattern, f) and f not in files.excludes:
-            systemd_units.append(f)
-
-    return systemd_units
-
-
-def write_sources(file):
-    """write out installs from SourceX lines"""
-    if len(sources["unit"]) != 0:
-        file.write_strip("mkdir -p %{buildroot}/usr/lib/systemd/system")
-        for unit in sources["unit"]:
-            file.write_strip("install -m 0644 %{{SOURCE{0}}} %{{buildroot}}/usr/lib/systemd/system/{1}"
-                             .format(source_index[unit], unit))
-    if len(sources["tmpfile"]) != 0:
-        file.write_strip("mkdir -p %{buildroot}/usr/lib/tmpfiles.d")
-        file.write_strip("install -m 0644 %{{SOURCE{0}}} %{{buildroot}}/usr/lib/tmpfiles.d/{1}.conf"
-                         .format(source_index[sources["tmpfile"][0]], tarball.name))
-
-
-def write_make_install_append(file):
-    """write out any custom supplied commands at the very end of the %install section"""
-    if make_install_append and make_install_append[0]:
-        file.write_strip("## make_install_append content")
-        for line in make_install_append:
-            file.write_strip("%s\n" % line)
-        file.write_strip("## make_install_append end")
-
-
-def write_systemd_units(file):
-    """write out installs for systemd unit files"""
-    units = get_systemd_units()
-    for unit in units:
-        file.write("systemctl --root=%{{buildroot}} enable {0}\n".format(os.path.basename(unit)))
-
-
-def write_buildpattern(file):
-    file.write_strip("\n")
-
-    pattern_method = globals().get('write_%s_pattern' % default_pattern, None)
-    if pattern_method:
-        pattern_method(file)
-
-    write_sources(file)
-    write_make_install_append(file)
-    # write_systemd_units(file)

autospec/check.py

@@ -0,0 +1,178 @@
+#!/bin/true
+#
+# test.py - part of autospec
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Deduce and emmit the patterns for %check
+#
+
+import os
+import re
+
+import count
+import util
+
+tests_config = ""
+
+
+def check_regression(pkg_dir, skip_tests, test_round):
+    """Check the build log for test regressions using the count module."""
+    if skip_tests:
+        return
+
+    log_path = os.path.join(pkg_dir, 'results', 'build.log')
+    result = count.parse_log(log_path)
+    if len(result) == 0 or result[0:2] == ',0':
+        log_path = os.path.join(pkg_dir, 'results', f"round{test_round}-build.log")
+        result = count.parse_log(log_path)
+
+    titles = [('Package', 'package name', 1),
+              ('Total', 'total tests', 1),
+              ('Pass', 'total passing', 1),
+              ('Fail', 'total failing', 0),
+              ('Skip', 'tests skipped', 0),
+              ('XFail', 'expected fail', 0)]
+    res_str = ""
+    for line in result.strip('\n').split('\n'):
+        s_line = line.split(',')
+        for idx, title in enumerate(titles):
+            if s_line[idx]:
+                if (s_line[idx] != '0') or (title[2] > 0):
+                    print("{}: {}".format(title[1], s_line[idx]))
+                res_str += "{} : {}\n".format(title[0], s_line[idx])
+
+    util.write_out(os.path.join(pkg_dir, "testresults"), res_str)
+
+
+def scan_for_tests(src_dir, config, requirements, content):
+    """Scan source directory for test files and set tests_config accordingly."""
+    global tests_config
+
+    if config.config_opts.get('skip_tests') or tests_config:
+        return
+
+    makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
+    make_check = "make {}check".format(makeflags)
+    cmake_check = "make test"
+    make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                         "make {}check\nmodule unload openmpi".format(makeflags)
+    cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                          "make test\nmodule unload openmpi"
+
+    if config.config_opts.get('allow_test_failures'):
+        make_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                             "make {}check || :\nmodule unload openmpi".format(makeflags)
+        cmake_check_openmpi = "module load openmpi\nexport OMPI_MCA_rmaps_base_oversubscribe=1\n" \
+                              "make test || :\nmodule unload openmpi"
+
+    perl_check = "make TEST_VERBOSE=1 test"
+    setup_check = """PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") python setup.py test"""
+    meson_check = "meson test -C builddir --print-errorlogs"
+    if config.config_opts.get('allow_test_failures'):
+        make_check += " || :"
+        cmake_check += " || :"
+        perl_check += " || :"
+        setup_check += " || :"
+        meson_check += " || :"
+
+    testsuites = {
+        "makecheck": make_check,
+        "perlcheck": perl_check,
+        "setup.py": setup_check,
+        "cmake": "cd clr-build; " + cmake_check,
+        "meson": meson_check,
+    }
+    if config.config_opts.get('32bit'):
+        testsuites["makecheck"] += "\ncd ../build32;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../clr-build32;\n" + cmake_check + " || :"
+        testsuites["meson"] += "\ncd ../build32;\n" + meson_check + " || :"
+    if config.config_opts.get('use_avx2'):
+        testsuites["makecheck"] += "\ncd ../buildavx2;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../clr-build-avx2;\n" + cmake_check + " || :"
+    if config.config_opts.get('use_avx512'):
+        testsuites["makecheck"] += "\ncd ../buildavx512;\n" + make_check + " || :"
+        testsuites["cmake"] += "\ncd ../clr-build-avx512;\n" + cmake_check + " || :"
+    if config.config_opts.get('openmpi'):
+        testsuites["makecheck"] += "\ncd ../build-openmpi;\n" + make_check_openmpi
+        testsuites["cmake"] += "\ncd ../clr-build-openmpi;\n" + cmake_check_openmpi
+
+    files = os.listdir(src_dir)
+
+    if config.default_pattern == "cmake":
+        makefile_path = os.path.join(src_dir, "CMakeLists.txt")
+        if not os.path.isfile(makefile_path):
+            return
+
+        if "enable_testing" in util.open_auto(makefile_path).read():
+            tests_config = testsuites["cmake"]
+
+    elif config.default_pattern in ["cpan", "configure", "configure_ac", "autogen"] and "Makefile.in" in files:
+        makefile_path = os.path.join(src_dir, "Makefile.in")
+        if os.path.isfile(makefile_path):
+            with util.open_auto(makefile_path, 'r') as make_fp:
+                lines = make_fp.readlines()
+            for line in lines:
+                if line.startswith("check:"):
+                    tests_config = testsuites["makecheck"]
+                    break
+                if line.startswith("test:"):
+                    tests_config = testsuites["perlcheck"]
+                    break
+
+    elif config.default_pattern in ["configure", "configure_ac", "autogen"] and "Makefile.am" in files:
+        tests_config = testsuites["makecheck"]
+
+    elif config.default_pattern in ["cpan"] and "Makefile.PL" in files:
+        tests_config = testsuites["perlcheck"]
+
+    elif config.default_pattern == "distutils3" and "setup.py" in files:
+        with util.open_auto(os.path.join(src_dir, "setup.py"), 'r') as setup_fp:
+            setup_contents = setup_fp.read()
+
+        if "test_suite" in setup_contents or "pbr=True" in setup_contents:
+            tests_config = testsuites["setup.py"]
+
+    elif config.default_pattern == "R":
+        tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\n"              \
+                       "R CMD check --no-manual --no-examples --no-codoc . "    \
+                       "|| :"
+    elif config.default_pattern == "meson":
+        found_tests = False
+        makefile_path = os.path.join(src_dir, "meson.build")
+        if not os.path.isfile(makefile_path):
+            return
+        for dirpath, _, files in os.walk(src_dir):
+            for f in files:
+                if f == "meson.build":
+                    with util.open_auto(os.path.join(dirpath, f)) as fp:
+                        if any(re.search(r'^\s*test\s*\(.+', line) for line in fp):
+                            found_tests = True
+                            tests_config = testsuites["meson"]
+                            break
+            if found_tests:
+                break
+
+    if "tox.ini" in files:
+        requirements.add_buildreq("pypi-tox")
+        requirements.add_buildreq("pypi-pytest")
+        requirements.add_buildreq("pypi-virtualenv")
+        requirements.add_buildreq("pypi-pluggy")
+        requirements.add_buildreq("pypi(py)")
+
+
+def load_specfile(specfile):
+    """Load the specfile object."""
+    specfile.tests_config = tests_config

autospec/cmake_modules

@@ -0,0 +1,972 @@
+# This file maps the CMake FIND_PACKAGE name to the packaage name, to be
+# added to BuildRequires. Please keep the list sorted
+# Format: <cmake modulename>, <pkg name(s)>
+2Geom, inkscape-dev
+ALSA, alsa-lib-dev
+ASPELL, aspell-dev
+ATen, pypi(pytorch)
+AccountsQt5, libaccounts-qt-dev
+Alembic, scene-alembic-dev
+Analitza5, analitza-dev
+AppStreamQt, appstream-dev
+Arcus, libArcus-dev
+Argon2, argon2-dev
+Armadillo, armadillo-data
+Astro, marble-dev
+BISON, bison-dev
+BLAS, openblas
+BZip2, bzip2-dev
+Backtrace, glibc-dev
+Boost, boost-dev
+Breeze, breeze-dev
+CAF, zeek-dev
+CGAL, CGAL-dev
+CGALLib, CGAL-dev
+CGAL_CoreLib, CGAL-dev
+CGAL_ImageIOLib, CGAL-dev
+CGAL_Qt5Lib, CGAL-dev
+CLucene, clucene-core
+CURL, curl-dev
+Caffe2, pypi(pytorch)
+Cantor, cantor-dev
+CapnProto, capnproto-dev
+CheckFlagCommon, cmake-data
+Clang, llvm-dev
+Codec2, codec2-dev
+CommonCMake, zeek-data
+Cups, cups-dev
+Curses, ncurses-dev
+DBus1, dbus-dev
+DigikamCore, digikam-dev
+DigikamDatabase, digikam-dev
+DigikamGui, digikam-dev
+DigikamPlugin, digikam-dev
+DolphinVcs, dolphin-dev
+Doxygen, doxygen
+ECM, extra-cmake-modules-data
+EGL, extra-cmake-modules pkgconfig(egl)
+EXPAT, expat-dev
+Ecore, efl-dev
+EcoreCxx, efl-dev
+Edje, efl-dev
+Eet, efl-dev
+EetCxx, efl-dev
+Eeze, efl-dev
+Efl, efl-dev
+Efreet, efl-dev
+Eigen3, eigen-data
+Eina, efl-dev
+EinaCxx, efl-dev
+Eio, efl-dev
+Eldbus, efl-dev
+Elementary, efl-dev
+Elua, efl-dev
+Emile, efl-dev
+Emotion, efl-dev
+Eo, efl-dev
+EoCxx, efl-dev
+Eolian, efl-dev
+EolianCxx, efl-dev
+Ethumb, efl-dev
+EthumbClient, efl-dev
+Evas, efl-dev
+EvasCxx, efl-dev
+FLEX, flex
+FLTK, fltk-dev
+FindFont, VTK-dev
+FindPkg, cmake-data
+FreeGLUT, freeglut-dev
+FreeRDP, FreeRDP-dev
+FreeRDP-Client, FreeRDP-dev
+FreeRDP-Server, FreeRDP-dev
+FreeRDP-Shadow, FreeRDP-dev
+Freetype, freetype-dev
+GDAL, gdal-dev
+GLEW, glew-dev
+GLIB2, extra-cmake-modules pkgconfig(glib-2.0)
+GLU, glu-dev
+GLUT, freeglut-dev
+GSL, gsl-dev
+GTK2, gtk+-dev
+GTest, googletest-dev
+Gcrypt, libgcrypt-dev
+Gettext, gettext-dev
+Git, git
+Gloo, pypi(pytorch)
+Gmic, gmic-dev
+GnuTLS, gnutls-dev
+Gnuplot, gnuplot-dev
+Gperf, extra-cmake-modules gperf
+Gpgmepp, gpgme-dev
+Grantlee5, grantlee-dev
+GtkDoc, gtk-doc-data
+HDF5, hdf5-dev
+ICU, icu4c-dev
+Iconv, glibc-dev
+IlmBase, openexr-dev
+Imath, Imath-dev
+InferenceEngine, dldt-dev
+Inotify, extra-cmake-modules
+Intl, glibc-dev
+JNI, openjdk-dev
+JPEG, libjpeg-turbo-dev
+Java, openjdk
+KAccounts, kaccounts-integration-dev
+KChart, kdiagram-dev
+KDED, kded-dev
+KDEExperimentalPurpose, purpose-dev
+KDSoap, kdsoap-dev
+KDecoration2, kdecoration-dev
+KF5Activities, kactivities-dev
+KF5ActivitiesStats, kactivities-stats-dev
+KF5Akonadi, akonadi-dev
+KF5AkonadiCalendar, akonadi-calendar-dev
+KF5AkonadiContact, akonadi-contacts-dev
+KF5AkonadiMime, akonadi-mime-dev
+KF5AkonadiNotes, akonadi-notes-dev
+KF5AkonadiSearch, akonadi-search-dev
+KF5AlarmCalendar, kalarmcal-dev
+KF5Archive, karchive-dev
+KF5Attica, attica-dev
+KF5Auth, kauth-dev
+KF5Baloo, baloo-dev
+KF5BalooWidgets, baloo-widgets-dev
+KF5Blog, kblog-dev
+KF5BluezQt, bluez-qt-dev
+KF5Bookmarks, kbookmarks-dev
+KF5CalendarCore, kcalendarcore-dev
+KF5CalendarSupport, calendarsupport-dev
+KF5CalendarUtils, kcalutils-dev
+KF5Cddb, libkcddb-dev
+KF5Codecs, kcodecs-dev
+KF5CompactDisc, libkcompactdisc-dev
+KF5Completion, kcompletion-dev
+KF5Config, kconfig-dev
+KF5ConfigWidgets, kconfigwidgets-dev
+KF5ContactEditor, akonadi-contacts-dev
+KF5Contacts, kcontacts-dev
+KF5CoreAddons, kcoreaddons-dev
+KF5Crash, kcrash-dev
+KF5DAV, kdav-dev
+KF5DBusAddons, kdbusaddons-dev
+KF5DNSSD, kdnssd-dev
+KF5Declarative, kdeclarative-dev
+KF5DesignerPlugin, kdesignerplugin-dev
+KF5DocTools, kdoctools-dev
+KF5Emoticons, kemoticons-dev
+KF5EventViews, eventviews-dev
+KF5FileMetaData, kfilemetadata-dev
+KF5FollowupReminder, kdepim-apps-libs-dev
+KF5FrameworkIntegration, frameworkintegration-dev
+KF5GlobalAccel, kglobalaccel-dev
+KF5GrantleeTheme, grantleetheme-dev
+KF5Gravatar, libgravatar-dev
+KF5GuiAddons, kguiaddons-dev
+KF5Holidays, kholidays-dev
+KF5I18n, ki18n-dev
+KF5IMAP, kimap-dev
+KF5IconThemes, kiconthemes-dev
+KF5IdentityManagement, kidentitymanagement-dev
+KF5IdleTime, kidletime-dev
+KF5Init, kinit-dev
+KF5ItemModels, kitemmodels-dev
+KF5ItemViews, kitemviews-dev
+KF5JS, kjs-dev
+KF5JobWidgets, kjobwidgets-dev
+KF5JsEmbed, kjsembed-dev
+KF5KCMUtils, kcmutils-dev
+KF5KDE4Support, kdelibs4support-dev
+KF5KDEGames, libkdegames-dev
+KF5KDELibs4Support, kdelibs4support-dev
+KF5KDcraw, libkdcraw-dev
+KF5KExiv2, libkexiv2-dev
+KF5KHtml, khtml-dev
+KF5KIO, kio-dev
+KF5KMahjongglib, libkmahjongg-dev
+KF5KaddressbookGrantlee, kdepim-apps-libs-dev
+KF5KaddressbookImportExport, kdepim-apps-libs-dev
+KF5KdepimDBusInterfaces, kdepim-apps-libs-dev
+KF5Kipi, libkipi-dev
+KF5Kirigami2, kirigami2-dev
+KF5Konq, konqueror-dev
+KF5KontactInterface, kontactinterface-dev
+KF5Kross, kross-dev
+KF5Ldap, kldap-dev
+KF5LibKSieve, libksieve-dev
+KF5Libkdepim, libkdepim-dev
+KF5LibkdepimAkonadi, libkdepim-dev
+KF5Libkleo, libkleo-dev
+KF5MailCommon, mailcommon-dev
+KF5MailImporter, mailimporter-dev
+KF5MailImporterAkonadi, mailimporter-dev
+KF5MailTransport, kmailtransport-dev
+KF5MailTransportAkonadi, kmailtransport-dev
+KF5Mbox, kmbox-dev
+KF5MediaPlayer, kmediaplayer-dev
+KF5MessageComposer, messagelib-dev
+KF5MessageCore, messagelib-dev
+KF5MessageList, messagelib-dev
+KF5MessageViewer, messagelib-dev
+KF5Mime, kmime-dev
+KF5MimeTreeParser, messagelib-dev
+KF5ModemManagerQt, modemmanager-qt-dev
+KF5NetworkManagerQt, networkmanager-qt-dev
+KF5NewStuff, knewstuff-dev
+KF5NewStuffCore, knewstuff-dev
+KF5NewStuffQuick, knewstuff-dev
+KF5Notifications, knotifications-dev
+KF5NotifyConfig, knotifyconfig-dev
+KF5Package, kpackage-dev
+KF5Parts, kparts-dev
+KF5People, kpeople-dev
+KF5PimCommon, pimcommon-dev
+KF5PimCommonAkonadi, pimcommon-dev
+KF5PimTextEdit, kpimtextedit-dev
+KF5Plasma, plasma-framework-dev
+KF5PlasmaQuick, plasma-framework-dev
+KF5Plotting, kplotting-dev
+KF5Prison, prison-dev
+KF5Pty, kpty-dev
+KF5Purpose, purpose-dev
+KF5QQC2DeskopStyle, qqc2-desktop-style-dev
+KF5QQC2DesktopStyle, qqc2-desktop-style-dev
+KF5QuickCharts, kquickcharts-dev
+KF5Runner, krunner-dev
+KF5Sane, libksane-dev
+KF5Screen, libkscreen-dev
+KF5SendLater, kdepim-apps-libs-dev
+KF5Service, kservice-dev
+KF5Solid, solid-dev
+KF5Sonnet, sonnet-dev
+KF5Su, kdesu-dev
+KF5Syndication, syndication-dev
+KF5SyntaxHighlighting, syntax-highlighting-dev
+KF5SysGuard, libksysguard-dev
+KF5TemplateParser, messagelib-dev
+KF5TextEditor, ktexteditor-dev
+KF5TextWidgets, ktextwidgets-dev
+KF5ThreadWeaver, threadweaver-dev
+KF5Tnef, ktnef-dev
+KF5UnitConversion, kunitconversion-dev
+KF5Wallet, kwallet-dev
+KF5Wayland, kwayland-dev
+KF5WebEngineViewer, messagelib-dev
+KF5WidgetsAddons, kwidgetsaddons-dev
+KF5WindowSystem, kwindowsystem-dev
+KF5XmlGui, kxmlgui-dev
+KF5XmlRpcClient, kxmlrpcclient-dev
+KGantt, kdiagram-dev
+KHotKeysDBusInterface, khotkeys-dev
+KPimGAPI, libkgapi-dev
+KPimItinerary, kitinerary-dev
+KPimPkPass, kpkpass-dev
+KPimSMTP, ksmtp-dev
+KRunnerAppDBusInterface, plasma-workspace-dev
+KSMServerDBusInterface, plasma-workspace-dev
+KSaneCore, ksanecore-dev
+KScreenLocker, kscreenlocker-dev
+KSysGuard, libksysguard-dev
+KUserFeedback, kuserfeedback-dev
+KWaylandServer, kwayland-server-dev
+KWinDBusInterface, kwin-dev
+KWinEffects, kwin-dev
+KastenControllers, okteta-dev
+KastenCore, okteta-dev
+KastenGui, okteta-dev
+KioArchive, kio-extras-dev
+LATEX, texlive
+LLD, llvm-dev
+LLVM, llvm-dev
+LLVM-, llvm-dev
+LayerShellQt, layer-shell-qt-dev
+Leptonica, leptonica-dev
+LibArchive, libarchive-dev
+LibColorCorrect, plasma-workspace-dev
+LibFTDI1, libftdi1-dev
+LibGPGError, libgpg-error-dev
+LibGit2, extra-cmake-modules pkgconfig(libgit2)
+LibIcal, libical-dev
+LibKEduVocDocument, libkeduvocdocument-dev
+LibKWorkspace, plasma-workspace-dev
+LibKompareDiff2, libkomparediff2-dev
+LibNotificationManager, plasma-workspace-dev
+LibObs, obs-studio-dev
+LibTaskManager, plasma-workspace-dev
+LibXml2, libxml2-dev
+LibXslt, libxslt-dev
+LibtorrentRasterbar, libtorrent-rasterbar-dev
+Lua, lua-dev
+MPI, openmpi-dev
+MailTransportDBusService, libkdepim-dev
+Marble, marble-dev
+Mlt7, mlt-dev
+Motif, motif-dev
+NLopt, nlopt-dev
+OCE, OpenCASCADE
+OktetaCore, okteta-dev
+OktetaGui, okteta-dev
+OktetaKastenControllers, okteta-dev
+OktetaKastenCore, okteta-dev
+OktetaKastenGui, okteta-dev
+Okular5, okular-dev
+OpenAL, openal-soft-dev
+OpenBLAS, openblas-dev
+OpenCV, opencv-dev
+OpenColorIO, OpenColorIO-data
+OpenEXR, extra-cmake-modules pkgconfig(OpenEXR)
+OpenGL, mesa-dev
+OpenImageIO, oiio-dev
+OpenJPEG, openjpeg
+OpenMP, cmake
+OpenSSL, openssl-dev
+PHP4, php-dev
+PNG, libpng-dev
+PahoMqttCpp, paho.mqtt.cpp-dev
+Pala, palapeli-dev
+Patch, patch
+Perl, perl
+PerlLibs, perl
+Phonon4Qt5, phonon-dev
+Phonon4Qt5Experimental, phonon-dev
+PkgConfig, pkg-config
+PlasmaActivities, plasma-activities-dev
+PlasmaPotdProvider, kdeplasma-addons-dev
+PlasmaWaylandProtocols, plasma-wayland-protocols-dev
+Png2Ico, extra-cmake-modules png2ico
+PolkitQt5-1, polkit-qt-dev
+Polly, llvm-dev
+Poppler, extra-cmake-modules pkgconfig(poppler)
+PortMidi, portmidi-dev
+PostgreSQL, postgresql-dev
+Protobuf, protobuf-dev
+PulseAudio, extra-cmake-modules pkgconfig(libpulse)
+PyIlmBase, openexr-dev
+PySide2, pyside2-setup-dev
+Python, python3
+Python3, python3-dev
+PythonInterp, python3
+PythonLibs, python3-dev
+QGpgme, gpgme-dev gpgme-extras
+QHelpGenerator, extra-cmake-modules qt6tools-dev
+QMobipocket, kdegraphics-mobipocket-dev
+Qca-qt5, qca-qt5-dev
+Qt3DTests, qt6base-dev
+Qt5, qtbase-dev mesa-dev
+Qt53DAnimation, qt3d-dev
+Qt53DCore, qt3d-dev
+Qt53DExtras, qt3d-dev
+Qt53DInput, qt3d-dev
+Qt53DLogic, qt3d-dev
+Qt53DQuick, qt3d-dev
+Qt53DQuickAnimation, qt3d-dev
+Qt53DQuickExtras, qt3d-dev
+Qt53DQuickInput, qt3d-dev
+Qt53DQuickRender, qt3d-dev
+Qt53DQuickScene2D, qt3d-dev
+Qt53DRender, qt3d-dev
+Qt5AccessibilitySupport, qtbase-dev
+Qt5AttributionsScannerTools, qttools-dev
+Qt5Bluetooth, qtconnectivity-dev
+Qt5Bootstrap, qtbase-dev
+Qt5Charts, qtcharts-dev
+Qt5CompatTests, qt6base-dev
+Qt5Concurrent, qtbase-dev
+Qt5Core, qtbase-dev
+Qt5DBus, qtbase-dev
+Qt5DataVisualization, qtdatavis3d-dev
+Qt5Designer, qttools-dev
+Qt5DesignerComponents, qttools-dev
+Qt5DeviceDiscoverySupport, qtbase-dev
+Qt5EdidSupport, qtbase-dev
+Qt5EglFSDeviceIntegration, qtbase-dev
+Qt5EglFsKmsSupport, qtbase-dev
+Qt5EglSupport, qtbase-dev
+Qt5EventDispatcherSupport, qtbase-dev
+Qt5FbSupport, qtbase-dev
+Qt5FontDatabaseSupport, qtbase-dev
+Qt5Gamepad, qtgamepad-dev
+Qt5GlxSupport, qtbase-dev
+Qt5Gui, qtbase-dev
+Qt5Help, qttools-dev
+Qt5InputSupport, qtbase-dev
+Qt5Keychain, qtkeychain-dev
+Qt5KmsSupport, qtbase-dev
+Qt5LinguistTools, qttools-dev
+Qt5Location, qtlocation-dev
+Qt5Mqtt, qtmqtt-dev
+Qt5Multimedia, qtmultimedia-dev
+Qt5MultimediaGstTools, qtmultimedia-dev
+Qt5MultimediaQuick, qtmultimedia-dev
+Qt5MultimediaWidgets, qtmultimedia-dev
+Qt5Network, qtbase-dev
+Qt5NetworkAuth, qtnetworkauth-dev
+Qt5Nfc, qtconnectivity-dev
+Qt5OpenGL, qtbase-dev
+Qt5OpenGLExtensions, qtbase-dev
+Qt5PacketProtocol, qtdeclarative-dev
+Qt5Pdf, qtwebengine-dev
+Qt5PdfWidgets, qtwebengine-dev
+Qt5PlatformCompositorSupport, qtbase-dev
+Qt5Positioning, qtlocation-dev
+Qt5PositioningQuick, qtlocation-dev
+Qt5PrintSupport, qtbase-dev
+Qt5Qml, qtdeclarative-dev
+Qt5QmlDebug, qtdeclarative-dev
+Qt5QmlDevTools, qtdeclarative-dev
+Qt5QmlImportScanner, qtdeclarative-dev
+Qt5QmlModels, qtdeclarative-dev
+Qt5QmlWorkerScript, qtdeclarative-dev
+Qt5Quick, qtdeclarative-dev
+Qt5QuickCompiler, qtdeclarative-dev
+Qt5QuickControls2, qtquickcontrols2-dev
+Qt5QuickParticles, qtdeclarative-dev
+Qt5QuickShapes, qtdeclarative-dev
+Qt5QuickTemplates2, qtquickcontrols2-dev
+Qt5QuickTest, qtdeclarative-dev
+Qt5QuickWidgets, qtdeclarative-dev
+Qt5RemoteObjects, qtremoteobjects-dev
+Qt5RepParser, qtremoteobjects-dev
+Qt5Script, qtscript-dev
+Qt5ScriptTools, qtscript-dev
+Qt5Scxml, qtscxml-dev
+Qt5Sensors, qtsensors-dev
+Qt5SerialBus, qtserialbus-dev
+Qt5SerialPort, qtserialport-dev
+Qt5ServiceSupport, qtbase-dev
+Qt5Sql, qtbase-dev
+Qt5Svg, qtsvg-dev
+Qt5Test, qtbase-dev
+Qt5TextToSpeech, qtspeech-dev
+Qt5ThemeSupport, qtbase-dev
+Qt5UiPlugin, qttools-dev
+Qt5UiTools, qttools-dev
+Qt5VirtualKeyboard, qtvirtualkeyboard-dev
+Qt5VulkanSupport, qtbase-dev
+Qt5WaylandClient, qtwayland-dev
+Qt5WaylandCompositor, qtwayland-dev
+Qt5WebChannel, qtwebchannel-dev
+Qt5WebEngine, qtwebengine-dev
+Qt5WebEngineCore, qtwebengine-dev
+Qt5WebEngineWidgets, qtwebengine-dev
+Qt5WebSockets, qtwebsockets-dev
+Qt5Widgets, qtbase-dev
+Qt5X11Extras, qtx11extras-dev
+Qt5XcbQpa, qtbase-dev
+Qt5XkbCommonSupport, qtbase-dev
+Qt5Xml, qtbase-dev
+Qt5XmlPatterns, qtxmlpatterns-dev
+Qt6, qt6base-dev
+Qt63DAnimation, qt6base-dev
+Qt63DCore, qt6base-dev
+Qt63DExtras, qt6base-dev
+Qt63DInput, qt6base-dev
+Qt63DLogic, qt6base-dev
+Qt63DQuick, qt6base-dev
+Qt63DQuickAnimation, qt6base-dev
+Qt63DQuickExtras, qt6base-dev
+Qt63DQuickInput, qt6base-dev
+Qt63DQuickRender, qt6base-dev
+Qt63DQuickScene2D, qt6base-dev
+Qt63DRender, qt6base-dev
+Qt6AssimpImporterPlugin, qt6base-dev
+Qt6AssimpSceneImportPlugin, qt6base-dev
+Qt6Bluetooth, qt6base-dev
+Qt6BluetoothTools, qt6base-dev
+Qt6BodymovinPrivate, qt6base-dev
+Qt6BuildInternals, qt6base-dev
+Qt6Charts, qt6base-dev
+Qt6ChartsQml, qt6base-dev
+Qt6Coap, qt6base-dev
+Qt6Concurrent, qt6base-dev
+Qt6Core, qt6base-dev
+Qt6Core5Compat, qt6base-dev
+Qt6CoreTools, qt6base-dev
+Qt6DBus, qt6base-dev
+Qt6DBusTools, qt6base-dev
+Qt6DataVisualization, qt6base-dev
+Qt6DataVisualizationQml2, qt6base-dev
+Qt6DefaultGeometryLoaderPlugin, qt6base-dev
+Qt6Designer, qt6base-dev
+Qt6DesignerComponentsPrivate, qt6base-dev
+Qt6DeviceDiscoverySupportPrivate, qt6base-dev
+Qt6DmaBufServerBufferIntegrationPlugin, qt6base-dev
+Qt6DmaBufServerBufferPlugin, qt6base-dev
+Qt6DrmEglServerBufferIntegrationPlugin, qt6base-dev
+Qt6DrmEglServerBufferPlugin, qt6base-dev
+Qt6EglFSDeviceIntegrationPrivate, qt6base-dev
+Qt6EglFsKmsGbmSupportPrivate, qt6base-dev
+Qt6EglFsKmsSupportPrivate, qt6base-dev
+Qt6FbSupportPrivate, qt6base-dev
+Qt6GLTFGeometryLoaderPlugin, qt6base-dev
+Qt6GLTFSceneExportPlugin, qt6base-dev
+Qt6GLTFSceneImportPlugin, qt6base-dev
+Qt6Gui, qt6base-dev
+Qt6GuiTools, qt6base-dev
+Qt6Help, qt6base-dev
+Qt6HostInfo, qt6base-dev
+Qt6HunspellInputMethodPrivate, qt6base-dev
+Qt6IIOSensorProxySensorPlugin, qt6base-dev
+Qt6InputSupportPrivate, qt6base-dev
+Qt6KmsSupportPrivate, qt6base-dev
+Qt6LabsAnimation, qt6base-dev
+Qt6LabsFolderListModel, qt6base-dev
+Qt6LabsQmlModels, qt6base-dev
+Qt6LabsSettings, qt6base-dev
+Qt6LabsSharedImage, qt6base-dev
+Qt6LabsWavefrontMesh, qt6base-dev
+Qt6Linguist, qt6base-dev
+Qt6LinguistTools, qt6base-dev
+Qt6Mqtt, qt6base-dev
+Qt6Multimedia, qt6base-dev
+Qt6MultimediaQuickPrivate, qt6base-dev
+Qt6MultimediaWidgets, qt6base-dev
+Qt6Network, qt6base-dev
+Qt6NetworkAuth, qt6base-dev
+Qt6Nfc, qt6base-dev
+Qt6OpcUa, qt6base-dev
+Qt6OpenGL, qt6base-dev
+Qt6OpenGLRendererPlugin, qt6base-dev
+Qt6OpenGLWidgets, qt6base-dev
+Qt6PacketProtocolPrivate, qt6base-dev
+Qt6PassThruCanBusPlugin, qt6base-dev
+Qt6PeakCanBusPlugin, qt6base-dev
+Qt6Positioning, qt6base-dev
+Qt6PositioningQuick, qt6base-dev
+Qt6PrintSupport, qt6base-dev
+Qt6QComposePlatformInputContextPlugin, qt6base-dev
+Qt6QCupsPrinterSupportPlugin, qt6base-dev
+Qt6QDebugMessageServiceFactoryPlugin, qt6base-dev
+Qt6QEglFSEmulatorIntegrationPlugin, qt6base-dev
+Qt6QEglFSIntegrationPlugin, qt6base-dev
+Qt6QEglFSKmsEglDeviceIntegrationPlugin, qt6base-dev
+Qt6QEglFSKmsGbmIntegrationPlugin, qt6base-dev
+Qt6QEglFSX11IntegrationPlugin, qt6base-dev
+Qt6QEvdevKeyboardPlugin, qt6base-dev
+Qt6QEvdevMousePlugin, qt6base-dev
+Qt6QEvdevTabletPlugin, qt6base-dev
+Qt6QEvdevTouchScreenPlugin, qt6base-dev
+Qt6QGeoPositionInfoSourceFactoryGeoclue2Plugin, qt6base-dev
+Qt6QGeoPositionInfoSourceFactoryNmeaPlugin, qt6base-dev
+Qt6QGeoPositionInfoSourceFactoryPollPlugin, qt6base-dev
+Qt6QGifPlugin, qt6base-dev
+Qt6QGtk3ThemePlugin, qt6base-dev
+Qt6QICNSPlugin, qt6base-dev
+Qt6QICOPlugin, qt6base-dev
+Qt6QIbusPlatformInputContextPlugin, qt6base-dev
+Qt6QJpegPlugin, qt6base-dev
+Qt6QLibInputPlugin, qt6base-dev
+Qt6QLinuxFbIntegrationPlugin, qt6base-dev
+Qt6QLocalClientConnectionFactoryPlugin, qt6base-dev
+Qt6QMYSQLDriverPlugin, qt6base-dev
+Qt6QMinimalEglIntegrationPlugin, qt6base-dev
+Qt6QMinimalIntegrationPlugin, qt6base-dev
+Qt6QNetworkManagerNetworkInformationPlugin, qt6base-dev
+Qt6QOffscreenIntegrationPlugin, qt6base-dev
+Qt6QOpen62541Plugin, qt6base-dev
+Qt6QPSQLDriverPlugin, qt6base-dev
+Qt6QQmlDebugServerFactoryPlugin, qt6base-dev
+Qt6QQmlDebuggerServiceFactoryPlugin, qt6base-dev
+Qt6QQmlInspectorServiceFactoryPlugin, qt6base-dev
+Qt6QQmlNativeDebugConnectorFactoryPlugin, qt6base-dev
+Qt6QQmlNativeDebugServiceFactoryPlugin, qt6base-dev
+Qt6QQmlPreviewServiceFactoryPlugin, qt6base-dev
+Qt6QQmlProfilerServiceFactoryPlugin, qt6base-dev
+Qt6QQuickProfilerAdapterFactoryPlugin, qt6base-dev
+Qt6QQuickWidgetPlugin, qt6base-dev
+Qt6QSQLiteDriverPlugin, qt6base-dev
+Qt6QScxmlEcmaScriptDataModelPlugin, qt6base-dev
+Qt6QSvgIconPlugin, qt6base-dev
+Qt6QSvgPlugin, qt6base-dev
+Qt6QTcpServerConnectionFactoryPlugin, qt6base-dev
+Qt6QTgaPlugin, qt6base-dev
+Qt6QTiffPlugin, qt6base-dev
+Qt6QTlsBackendCertOnlyPlugin, qt6base-dev
+Qt6QTlsBackendOpenSSLPlugin, qt6base-dev
+Qt6QTuioTouchPlugin, qt6base-dev
+Qt6QVirtualKeyboardPlugin, qt6base-dev
+Qt6QVkKhrDisplayIntegrationPlugin, qt6base-dev
+Qt6QVncIntegrationPlugin, qt6base-dev
+Qt6QWaylandBradientDecorationPlugin, qt6base-dev
+Qt6QWaylandDmabufClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandEglClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandEglClientBufferPlugin, qt6base-dev
+Qt6QWaylandEglPlatformIntegrationPlugin, qt6base-dev
+Qt6QWaylandEglStreamClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandFullScreenShellV1IntegrationPlugin, qt6base-dev
+Qt6QWaylandIntegrationPlugin, qt6base-dev
+Qt6QWaylandIviShellIntegrationPlugin, qt6base-dev
+Qt6QWaylandWlShellIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeEglClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeEglClientBufferPlugin, qt6base-dev
+Qt6QWaylandXCompositeEglPlatformIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeGlxClientBufferIntegrationPlugin, qt6base-dev
+Qt6QWaylandXCompositeGlxClientBufferPlugin, qt6base-dev
+Qt6QWaylandXCompositeGlxPlatformIntegrationPlugin, qt6base-dev
+Qt6QWaylandXdgShellIntegrationPlugin, qt6base-dev
+Qt6QWbmpPlugin, qt6base-dev
+Qt6QWebpPlugin, qt6base-dev
+Qt6QXcbEglIntegrationPlugin, qt6base-dev
+Qt6QXcbGlxIntegrationPlugin, qt6base-dev
+Qt6QXcbIntegrationPlugin, qt6base-dev
+Qt6QXdgDesktopPortalThemePlugin, qt6base-dev
+Qt6Qml, qt6base-dev
+Qt6QmlCompilerPrivate, qt6base-dev
+Qt6QmlCore, qt6base-dev
+Qt6QmlDebugPrivate, qt6base-dev
+Qt6QmlDevToolsPrivate, qt6base-dev
+Qt6QmlDomPrivate, qt6base-dev
+Qt6QmlImportScanner, qt6base-dev
+Qt6QmlLocalStorage, qt6base-dev
+Qt6QmlModels, qt6base-dev
+Qt6QmlTools, qt6base-dev
+Qt6QmlWorkerScript, qt6base-dev
+Qt6QmlXmlListModel, qt6base-dev
+Qt6QtVirtualKeyboardHangulPlugin, qt6base-dev
+Qt6QtVirtualKeyboardHunspellPlugin, qt6base-dev
+Qt6QtVirtualKeyboardOpenWnnPlugin, qt6base-dev
+Qt6QtVirtualKeyboardPinyinPlugin, qt6base-dev
+Qt6QtVirtualKeyboardTCImePlugin, qt6base-dev
+Qt6QtVirtualKeyboardThaiPlugin, qt6base-dev
+Qt6Quick, qt6base-dev
+Qt6Quick3D, qt6base-dev
+Qt6Quick3DAssetImport, qt6base-dev
+Qt6Quick3DAssetUtils, qt6base-dev
+Qt6Quick3DEffects, qt6base-dev
+Qt6Quick3DHelpers, qt6base-dev
+Qt6Quick3DIblBaker, qt6base-dev
+Qt6Quick3DParticles, qt6base-dev
+Qt6Quick3DRuntimeRender, qt6base-dev
+Qt6Quick3DTools, qt6base-dev
+Qt6Quick3DUtils, qt6base-dev
+Qt6QuickControls2, qt6base-dev
+Qt6QuickControls2Impl, qt6base-dev
+Qt6QuickControlsTestUtilsPrivate, qt6base-dev
+Qt6QuickDialogs2, qt6base-dev
+Qt6QuickDialogs2QuickImpl, qt6base-dev
+Qt6QuickDialogs2Utils, qt6base-dev
+Qt6QuickLayouts, qt6base-dev
+Qt6QuickParticlesPrivate, qt6base-dev
+Qt6QuickShapesPrivate, qt6base-dev
+Qt6QuickTemplates2, qt6base-dev
+Qt6QuickTest, qt6base-dev
+Qt6QuickTestUtilsPrivate, qt6base-dev
+Qt6QuickTestplugin, qt6base-dev
+Qt6QuickTimeline, qt6base-dev
+Qt6QuickWidgets, qt6base-dev
+Qt6RemoteObjects, qt6base-dev
+Qt6RemoteObjectsQml, qt6base-dev
+Qt6RemoteObjectsTools, qt6base-dev
+Qt6RepParser, qt6base-dev
+Qt6Scene2DPlugin, qt6base-dev
+Qt6Scxml, qt6base-dev
+Qt6ScxmlQml, qt6base-dev
+Qt6ScxmlTools, qt6base-dev
+Qt6Sensors, qt6base-dev
+Qt6SensorsQuick, qt6base-dev
+Qt6SensorsQuickplugin, qt6base-dev
+Qt6SerialBus, qt6base-dev
+Qt6SerialBusTools, qt6base-dev
+Qt6SerialPort, qt6base-dev
+Qt6ShaderTools, qt6shadertools-dev
+Qt6ShaderToolsTools, qt6shadertools-dev
+Qt6ShmServerBufferIntegrationPlugin, qt6base-dev
+Qt6ShmServerBufferPlugin, qt6base-dev
+Qt6SocketCanBusPlugin, qt6base-dev
+Qt6Sql, qt6base-dev
+Qt6StateMachine, qt6base-dev
+Qt6StateMachineQml, qt6base-dev
+Qt6Svg, qt6base-dev
+Qt6SvgWidgets, qt6base-dev
+Qt6Test, qt6base-dev
+Qt6TinyCanBusPlugin, qt6base-dev
+Qt6Tools, qt6base-dev
+Qt6ToolsTools, qt6base-dev
+Qt6UiPlugin, qt6base-dev
+Qt6UiTools, qt6base-dev
+Qt6UipAssetImporterPlugin, qt6base-dev
+Qt6VirtualCanBusPlugin, qt6base-dev
+Qt6VirtualKeyboard, qt6base-dev
+Qt6VulkanServerBufferIntegrationPlugin, qt6base-dev
+Qt6VulkanServerBufferPlugin, qt6base-dev
+Qt6WaylandClient, qt6base-dev
+Qt6WaylandCompositor, qt6base-dev
+Qt6WaylandCompositorIviapplication, qt6base-dev
+Qt6WaylandCompositorWLShell, qt6base-dev
+Qt6WaylandCompositorXdgShell, qt6base-dev
+Qt6WaylandEglClientHwIntegrationPrivate, qt6base-dev
+Qt6WaylandEglCompositorHwIntegrationPrivate, qt6base-dev
+Qt6WaylandScannerTools, qt6base-dev
+Qt6WaylandTextureSharing, qt6base-dev
+Qt6WaylandTextureSharingExtension, qt6base-dev
+Qt6WebChannel, qt6base-dev
+Qt6WebSockets, qt6base-dev
+Qt6WebView, qt6base-dev
+Qt6WebViewQuick, qt6base-dev
+Qt6Widgets, qt6base-dev
+Qt6WidgetsTools, qt6base-dev
+Qt6WlShellIntegrationPrivate, qt6base-dev
+Qt6XcbQpaPrivate, qt6base-dev
+Qt6Xml, qt6base-dev
+Qt6declarative_opcua, qt6base-dev
+Qt6declarative_remoteobjects, qt6base-dev
+Qt6declarative_scxml, qt6base-dev
+Qt6dummySensorPlugin, qt6base-dev
+Qt6genericSensorPlugin, qt6base-dev
+Qt6labsanimationplugin, qt6base-dev
+Qt6labsmodelsplugin, qt6base-dev
+Qt6lottieqtplugin, qt6base-dev
+Qt6modelsplugin, qt6base-dev
+Qt6particlesplugin, qt6base-dev
+Qt6positioningquickplugin, qt6base-dev
+Qt6qmlfolderlistmodelplugin, qt6base-dev
+Qt6qmllocalstorageplugin, qt6base-dev
+Qt6qmlplugin, qt6base-dev
+Qt6qmlsettingsplugin, qt6base-dev
+Qt6qmlshapesplugin, qt6base-dev
+Qt6qmlwavefrontmeshplugin, qt6base-dev
+Qt6qmlwebsockets, qt6base-dev
+Qt6qmlxmllistmodelplugin, qt6base-dev
+Qt6qquick3dplugin, qt6base-dev
+Qt6qquicklayoutsplugin, qt6base-dev
+Qt6qtchartsqml2, qt6base-dev
+Qt6qtgraphicaleffectsplugin, qt6base-dev
+Qt6qtgraphicaleffectsprivate, qt6base-dev
+Qt6qtlabsplatformplugin, qt6base-dev
+Qt6qtqmlcoreplugin, qt6base-dev
+Qt6qtqmlstatemachine, qt6base-dev
+Qt6qtquick2plugin, qt6base-dev
+Qt6qtquick3dassetutilsplugin, qt6base-dev
+Qt6qtquick3deffectplugin, qt6base-dev
+Qt6qtquick3dhelpersplugin, qt6base-dev
+Qt6qtquick3dparticles3dplugin, qt6base-dev
+Qt6qtquickcontrols2basicstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2basicstyleplugin, qt6base-dev
+Qt6qtquickcontrols2fusionstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2fusionstyleplugin, qt6base-dev
+Qt6qtquickcontrols2imaginestyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2imaginestyleplugin, qt6base-dev
+Qt6qtquickcontrols2implplugin, qt6base-dev
+Qt6qtquickcontrols2materialstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2materialstyleplugin, qt6base-dev
+Qt6qtquickcontrols2nativestyleplugin, qt6base-dev
+Qt6qtquickcontrols2plugin, qt6base-dev
+Qt6qtquickcontrols2universalstyleimplplugin, qt6base-dev
+Qt6qtquickcontrols2universalstyleplugin, qt6base-dev
+Qt6qtquickdialogs2quickimplplugin, qt6base-dev
+Qt6qtquickdialogsplugin, qt6base-dev
+Qt6qtquickscene2dplugin, qt6base-dev
+Qt6qtquickscene3dplugin, qt6base-dev
+Qt6qtquicktemplates2plugin, qt6base-dev
+Qt6qtquicktimelineplugin, qt6base-dev
+Qt6qtquickvirtualkeyboardplugin, qt6base-dev
+Qt6qtquickvirtualkeyboardsettingsplugin, qt6base-dev
+Qt6qtquickvirtualkeyboardstylesplugin, qt6base-dev
+Qt6qtwebviewquickplugin, qt6base-dev
+Qt6quick3danimationplugin, qt6base-dev
+Qt6quick3dcoreplugin, qt6base-dev
+Qt6quick3dextrasplugin, qt6base-dev
+Qt6quick3dinputplugin, qt6base-dev
+Qt6quick3dlogicplugin, qt6base-dev
+Qt6quick3drenderplugin, qt6base-dev
+Qt6quickmultimedia, qt6base-dev
+Qt6quicktooling, qt6base-dev
+Qt6quickwindow, qt6base-dev
+Qt6qwaylandcompositorplugin, qt6base-dev
+Qt6sharedimageplugin, qt6base-dev
+Qt6webchannel, qt6base-dev
+Qt6workerscriptplugin, qt6base-dev
+QtBaseTests, qt6base-dev
+QtChartsTests, qt6base-dev
+QtCoapTests, qt6base-dev
+QtConnectivityTests, qt6base-dev
+QtDataVisualizationTests, qt6base-dev
+QtDeclarativeTests, qt6base-dev
+QtLocationTests, qt6base-dev
+QtLottieTests, qt6base-dev
+QtMqttTests, qt6base-dev
+QtMultimediaTests, qt6base-dev
+QtNetworkAuthTests, qt6base-dev
+QtOpcUaTests, qt6base-dev
+QtQuick3DTests, qt6base-dev
+QtQuickTimelineTests, qt6base-dev
+QtRemoteObjectsTests, qt6base-dev
+QtScxmlTests, qt6base-dev
+QtSensorsTests, qt6base-dev
+QtSerialBusTests, qt6base-dev
+QtSerialPortTests, qt6base-dev
+QtShaderToolsTests, qt6shadertools-dev
+QtSvgTests, qt6base-dev
+QtTesting, ParaView-dev
+QtToolsTests, qt6base-dev
+QtVirtualKeyboardTests, qt6base-dev
+QtWaylandScanner, extra-cmake-modules qtwayland-dev
+QtWaylandTests, qt6base-dev
+QtWebChannelTests, qt6base-dev
+QtWebSocketsTests, qt6base-dev
+QtWebViewTests, qt6base-dev
+QuaZip-Qt5, quazip-dev
+QuaZip5, quazip-dev
+RdKafka, librdkafka-dev
+Rinutils, rinutils-dev
+Ruby, ruby
+SDL, SDL-dev
+SDL_image, SDL_image-dev
+SDL_mixer, SDL_mixer-dev
+SDL_net, SDL_net-dev
+SDL_ttf, SDL_ttf-dev
+SFML, SFML-dev
+SPIRV-Headers, llvm-data
+SPIRV-Tools, SPIRV-Tools-dev
+SPIRV-Tools-link, SPIRV-Tools-dev
+SPIRV-Tools-lint, SPIRV-Tools-dev
+SPIRV-Tools-opt, SPIRV-Tools-dev
+SPIRV-Tools-reduce, SPIRV-Tools-dev
+SUNDIALS, sundials-dev
+SWIG, swig
+Sasl2, extra-cmake-modules pkgconfig(libsasl2)
+Savitar, libSavitar-dev
+ScreenSaverDBusInterface, kscreenlocker-dev
+Seccomp, extra-cmake-modules
+ShaderTools, qt6shadertools-dev
+SharedMimeInfo, extra-cmake-modules shared-mime-info
+Shiboken2, pyside2-setup-dev
+SignOnQt5, signond-dev
+Snappy, snappy-dev
+Subversion, subversion
+SymEngine, symengine-dev
+TCL, tcl-dev tk-dev
+TIFF, tiff-dev
+TclStub, tcl-dev
+Tclsh, tcl
+TelepathyQt5, telepathy-qt-dev
+TelepathyQt5Service, telepathy-qt-dev
+Threads, glibc-dev
+Torch, pypi(pytorch)
+UnixCommands, bash coreutils gzip
+UsePkg, cmake-data
+VPL, oneVPL-dev
+VPP, vpp-dev
+VTK, VTK-dev
+Vc, Vc-dev
+Volk, volk-dev
+Vulkan, Vulkan-Headers-dev Vulkan-Loader-dev Vulkan-Tools
+Waffle, waffle-dev
+Wayland, extra-cmake-modules pkgconfig(wayland-client)
+WaylandScanner, extra-cmake-modules wayland
+Wget, wget
+WinPR, FreeRDP-dev
+Wireshark, wireshark
+Wish, tk
+X11, libX11-dev libICE-dev libSM-dev libXau-dev libXcomposite-dev libXcursor-dev libXdamage-dev libXdmcp-dev libXext-dev libXfixes-dev libXft-dev libXi-dev libXinerama-dev libXi-dev libXmu-dev libXpm-dev libXrandr-dev libXrender-dev libXres-dev libXScrnSaver-dev libXt-dev libXtst-dev libXv-dev libXxf86vm-dev
+X11_XCB, extra-cmake-modules pkgconfig(x11-xcb)
+XCB, extra-cmake-modules pkgconfig(xcb) xcb-util-cursor-dev xcb-util-image-dev xcb-util-keysyms-dev xcb-util-renderutil-dev xcb-util-wm-dev xcb-util-dev
+Z3, Z3-dev
+ZLIB, zlib-dev
+ZXing, zxing-dev
+Zopfli, zopfli-dev
+ade, ade-data
+arpack-ng, arpack-ng-dev
+assimp, assimp-dev
+automotive-dlt, dlt-daemon-dev
+bash-completion, bash-completion-data
+box2d, box2d-dev
+cJSON, cJSON-dev
+catkin, catkin-data
+cereal, cereal-dev
+clFFT, clFFT-dev
+cmark, cmark-dev
+cmocka, cmocka-dev
+codec2, codec2-dev
+coin, Coin-dev
+cpprestsdk, cpprestsdk-dev
+cppzmq, cppzmq-data
+dbusmenu-qt5, libdbusmenu-qt-dev
+dlib, dlib-dev
+dnnl, mkl-dnn-dev
+double-conversion, double-conversion-dev
+eclipse-paho-mqtt-c, paho.mqtt.c-dev
+embree, embree-dev
+exiv2, exiv2-dev
+expat, expat-dev
+fann, fann-dev
+fm-qt, libfm-qt-data
+fmt, fmt-dev
+gflags, gflags-dev
+glfw3, glfw-dev
+glm, glm-dev
+glog, glog-dev
+glslang, glslang-data
+harfbuzz, harfbuzz-dev
+hiredis, hiredis-c-data
+igsc, igsc-dev
+json-c, json-c-dev
+kColorPicker, kcolorpicker-dev
+kImageAnnotator, kimageannotator-dev
+kf6.Config, kconfig-dev
+kf6.CoreAddons, kcoreaddons-dev
+kf6.Crash, kcrash-dev
+kf6.DBusAddons, kdbusaddons-dev
+kf6.FileMetaData, kfilemetadata-dev
+kf6.I18n, ki18n-dev
+kf6.JobWidgets, kjobwidgets-dev
+kf6.KIO, kio-dev
+kf6.Notifications, knotifications-dev
+kf6.Purpose, purpose-dev
+kf6.Runner, krunner-dev
+kf6.Service, kservice-dev
+kf6.StatusNotifierItem, kstatusnotifieritem-dev
+kim-api, kim-api-data
+leveldb, leveldb-dev
+libavif, libavif-dev
+libconfig++, libconfig-dev
+libconfig, libconfig-dev
+libcuckoo, libcuckoo-data
+libjpeg-turbo, libjpeg-turbo-dev
+libpala, palapeli
+libssh, libssh-dev
+libxml2, libxml2-dev
+libzip, libzip-dev
+lpcnetfreedv, LPCNet-dev
+lxqt, liblxqt-dev
+lxqt-build-tools, lxqt-build-tools-data
+lxqt-globalkeys, lxqt-globalkeys-dev
+lxqt-globalkeys-ui, lxqt-globalkeys-dev
+md4c, md4c-dev
+md4cHtml, md4c-dev
+msgpack, msgpack-c-dev
+nanoflann, nanoflann-dev
+paraview, ParaView-dev
+pugixml, pugixml-dev
+pybind11, pypi(pybind11)
+qt5xdg, libqtxdg-data
+qt5xdgiconloader, libqtxdg-data
+qt6.DBus, qt6base-dev
+qt6.Gui, qt6base-dev
+qt6.Widgets, qt6base-dev
+qtxdg-tools, qtxdg-tools-data
+rabbitmq-c, rabbitmq-c-dev
+realsense2, librealsense-dev
+realsense2-gl, librealsense-dev
+rttr, rttr-data
+sdl2, SDL2-dev
+sdl2_ttf, SDL2_ttf-dev
+spdlog, spdlog-dev
+spirv_cross_c, SPIRV-Cross-data
+spirv_cross_c_shared, SPIRV-Cross-data
+spirv_cross_core, SPIRV-Cross-data
+spirv_cross_cpp, SPIRV-Cross-data
+spirv_cross_glsl, SPIRV-Cross-data
+spirv_cross_hlsl, SPIRV-Cross-data
+spirv_cross_msl, SPIRV-Cross-data
+spirv_cross_reflect, SPIRV-Cross-data
+spirv_cross_util, SPIRV-Cross-data
+sysstat-qt5, libsysstat-data
+tinyxml2, tinyxml2-dev
+tsl-robin-map, robin-map-data
+utf8cpp, utf8cpp-dev
+uwac, FreeRDP-dev
+vtk, ParaView-dev
+vtk, VTK-dev
+x86-64, cmake
+xapian, xapian-core-dev
+yaml-cpp, yaml-cpp-dev

autospec/commitmessage.py

@@ -22,149 +22,279 @@
 #
 #
 
+import os
 import re
+import shutil
+import sys
+from subprocess import PIPE, run
 
-import build
+import util
-import config
-import tarball
-
-commitmessage = []
-
-cves = set()
-have_cves = False
-cvestring = ""
 
 
-def new_cve(cve):
+def scan_for_changes(download_path, directory, transforms):
-    global cvestring
+    """Scan for changelogs or news files in the file sources.
-    global have_cves
-    global cves
 
-    cves.add(cve)
+    Scan for changelogs or news files in the source code and copy them to download_path as their
-    have_cves = True
+    `transform`ed name. The file with the transformed name will later be parsed to find the
-    cvestring += " " + cve
+    commit message.
+    """
+    found = []
+    interests = transforms.keys()
+    for dirpath, dirnames, files in os.walk(directory, topdown=False):
+        hits = [x for x in files if x.lower() in interests and x.lower() not in found]
+        for item in hits:
+            source = os.path.join(dirpath, item)
+            target = os.path.join(download_path, transforms[item.lower()])
+            try:
+                shutil.copy(source, target)
+                os.chmod(target, 0o644)
+            except Exception as e:
+                print("Error copying file: {}".format(e))
+                sys.exit(1)
+            found.append(item)
 
 
-def process_NEWS(file):
+def is_header(lines, curindex):
-    global commitmessage
+    """Check if the current line is a section header.
-    news = []
-    start = 0
-    stop = 0
-    success = 0
 
-    if config.old_version is None or config.old_version == tarball.version:
+    Check if the current line is a section header by looking for a blank line before it or an
-        return
+    underline/section break (---) after it. Returns True for lines at the beginning or end of the
+    file.
+    """
+    if curindex == 0:
+        # treat the start of the file as a header
+        # this will not be caught by an IndexError because -1 is a valid index
+        return True
 
     try:
-        with open(build.download_path + "/" + file) as f:
+        return (not lines[curindex - 1]) or ('---' in lines[curindex + 1])
-            news = f.readlines()
+    except IndexError:
+        # end of file doesn't matter for starting a block and is an obvious end
+        # of a block
+        return True
+
+
+def find_in_line(pattern, line):
+    """Return True if the pattern is in the line, False otherwise."""
+    return bool(re.search(pattern, line))
+
+
+def process_NEWS(newsfile, old_version, name, version, download_path):
+    """Parse the newfile for relevent changes.
+
+    Look for changes and CVE fixes relevant to current version update. This information is returned
+    as a tuple: (commitmessage, cves).
+
+    A maximum of 15 lines from the newsfile is returned in the commitmessage.
+    If the newsfile information is truncated to 15 lines an additional line is
+    added "(NEWS truncated at 15 lines)"
+    """
+    commitmessage = []
+    cves = set()
+    start = 0
+    stop = 0
+    success = False
+    start_found = False
+
+    if old_version is None or old_version == version:
+        # no version update, so no information to search for in newsfile
+        return commitmessage, cves
+
+    try:
+        with util.open_auto(os.path.join(download_path, newsfile)) as f:
+            newslines = f.readlines()
     except EnvironmentError:
-        return
+        return commitmessage, cves
 
-    stop = len(news)
+    newslines = [news.rstrip('\n') for news in newslines]
-    if stop <= start:
-        return
 
-    i = start
+    # escape some values for use in regular expressions below
-    while i < stop:
+    escaped_curver = re.escape(version)
-        news[i] = news[i].strip('\n')
+    escaped_oldver = re.escape(old_version)
-        i = i + 1
+    escaped_tarname = re.escape(name)
 
-    i = start + 1
+    # these are patterns that define the beginning of a block of information
-    while i < stop - 1:
+    # regarding the current version.
-        if news[i] == config.old_version and news[i - 1] == "":
+    news_start = [r'Version.*{}'.format(escaped_curver),
-            stop = i - 1
+                  r'(v|- )?{}:?'.format(escaped_curver),
-            success = 1
+                  r'{}-{}:?'.format(escaped_tarname, escaped_curver),
-        if news[i] == "Overview of changes leading to " + config.old_version and news[i - 1] == "":
+                  r'{} 20'.format(escaped_curver)]
-            stop = i - 1
-            success = 1
-        if news[i] == config.old_version + ":" and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == tarball.version + ":" and news[i - 1] == "":
-            start = i
-        if news[i] == tarball.name + "-" + config.old_version + ":" and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == tarball.name + "-" + tarball.version + ":" and news[i - 1] == "":
-            start = i
-        if news[i] == "- " + config.old_version + ":" and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i] == "- " + tarball.version + ":" and news[i - 1] == "":
-            start = i
-        if news[i].find(config.old_version) >= 0 and news[i].find("*** Changes in ") >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].find(config.old_version) >= 0 and news[i].find("201") >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].lower().find(tarball.name + " " + config.old_version) >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].find(config.old_version) >= 0 and news[i].find("Version ") >= 0 and news[i - 1] == "":
-            stop = i - 1
-            success = 1
-        if news[i].find(tarball.version) >= 0 and news[i].find("Version ") >= 0 and news[i - 1] == "":
-            start = i
 
-        if news[i].find(config.old_version + ":") == 0:
+    # these are patterns that define the end of a block of information
-            stop = i - 1
+    # regarding the current version.
-            success = 1
+    news_end = [r'\*\*\* Changes in.*{}'.format(escaped_oldver),
-        if news[i] == config.old_version and news[i + 1].find("---") >= 0:
+                r'{}.*201'.format(escaped_oldver),
-            stop = i - 1
+                r'Version.*{}'.format(escaped_oldver),
-            success = 1
+                r'^Overview of changes leading to {}'.format(escaped_oldver),
-        if news[i] == tarball.version and news[i + 1].find("---") >= 0:
+                r'^{}(-| ){}:?'.format(escaped_tarname, escaped_oldver),
-            start = i
+                r'v?{}:?'.format(escaped_oldver)]
 
-        i = i + 1
+    for idx, news in enumerate(newslines):
+        # only check headers for begin and end patterns
+        if is_header(newslines, idx):
+            for pat in news_start:
+                if find_in_line(pat, news):
+                    start = idx
+                    start_found = True
+                    break
+            if start_found:
+                for pat in news_end:
+                    if find_in_line(pat, news):
+                        success = True
+                        stop = idx - 1  # stop before this header
+                        break
+            if start_found and success:
+                break
 
-    if success == 0:
+    if not success or stop <= start:
-        return
+        return commitmessage, cves
 
     # now search for CVEs
-    i = start
+    pat = re.compile(r"(CVE\-[0-9]+\-[0-9]+)")
-    pat = re.compile("(CVE\-[0-9]+\-[0-9]+)")
+    for news in newslines[start:stop]:
-    while i < stop and i < start:
+        match = pat.search(news)
-        match = pat.search(news[i])
         if match:
-                s = match.group(1)
+            s = match.group(1)
-                new_cve(s)
+            cves.add(s)
-        i = i + 1
+
+    # compile commitmessage to return
+    commitmessage.append("")
+    for news in newslines[start:min(start + 15, stop)]:
+        commitmessage.append(news)
+
+    if stop > start + 15:
+        # append message that news was truncated
+        commitmessage.extend(["", "(NEWS truncated at 15 lines)"])
 
     commitmessage.append("")
-    i = start
+    return commitmessage, cves
-    while i < stop and i < start + 15:
-        commitmessage.append(news[i])
-        i = i + 1
-    commitmessage.append("")
 
 
-def guess_commit_message():
+def process_git(giturl, oldversion, newversion, name):
-    global cvestring
+    """Check out a git tree and try to turn the git history into a commit message."""
+    oldtag = ""
+    guessed_oldtag = oldversion
+    newtag = ""
+    guessed_newtag = newversion
+
+    if len(giturl) < 1:
+        return ""
+    if oldversion == newversion:
+        return ""
+
+    run(["git", "-C", "results", "clone", giturl, name])
+    p = run(["git", "-C", "results/" + name, "tag"], stdout=PIPE)
+    tags = p.stdout.decode('utf-8').split('\n')
+
+    # Version strings will sometimes have a leading 'v', 'V', or
+    # '<packagename>-' prefix, or possibly a combination of these.
+    regex_template = r'^({}-)?[vV]?{{}}$'.format(re.escape(name))
+
+    for t in tags:
+        i = t.find(oldversion)
+        if i != -1:
+            guessed_oldtag = t
+        pat = regex_template.format(re.escape(oldversion))
+        if re.search(pat, t):
+            oldtag = t
+        i = t.find(newversion)
+        if i != -1:
+            guessed_newtag = t
+        pat = regex_template.format(re.escape(newversion))
+        if re.search(pat, t):
+            newtag = t
+
+    if oldtag == "":
+        oldtag = guessed_oldtag
+    if newtag == "":
+        newtag = guessed_newtag
+
+    p = run(["git", "-C", "results/" + name, "log", "--no-merges", oldtag + ".." + newtag], stdout=PIPE)
+    fulllog = p.stdout.decode('utf-8', errors='replace').split('\n')
+    # 'git shortlog' can accept any 'git log' output over stdin, so make sure
+    # it lacks merge commits, too.
+    p = run(["git", "-C", "results/" + name, "shortlog"], input=p.stdout, stdout=PIPE)
+    shortlog = p.stdout.decode('utf-8', errors='replace').split('\n')
+
+    if len(fulllog) < 15:
+        return fulllog
+    else:
+        return shortlog
+
+
+def guess_commit_message(keyinfo, config, content):
+    """Parse newsfile for a commit message.
+
+    Try and find a sane commit message for the newsfile. The commit message defaults to the
+    following for an updated version if no CVEs are fixed:
+
+    <tarball name>: Autospec creation for update from version <old> to version <new>
+
+    If CVEs are fixed:
+
+    <tarball name>: Fix for <cve>
+
+    And if the version does not change:
+
+    <tarball name>: Autospec creation for version <version>
+
+    Additional information is appended to the commitmessage depending on NEWS
+    and ChangeLog files and the presence of CVEs. The commitmessage is written
+    to a file at <download path>/commitmsg.
+    """
+    cvestring = ""
+    cves = set()
+    commitmessage = []
+    for cve in config.cves:
+        cves.add(cve)
+        cvestring += " " + cve
 
     # default commit messages before we get too smart
-    if config.old_version is not None and config.old_version != tarball.version:
+    if config.old_version is not None and config.old_version != content.version:
-        commitmessage.append(tarball.name + ": Autospec creation for update from version " +
+        commitmessage.append("{}: Autospec creation for update from version {} to version {}"
-                             config.old_version + " to version " +
+                             .format(content.name, config.old_version, content.version))
-                             tarball.version)
+        if content.giturl != "":
+            gitmsg = process_git(content.giturl, config.old_version, content.version, content.name)
+            commitmessage.append("")
+            commitmessage.extend(gitmsg)
     else:
-        if have_cves:
+        if cves:
-            commitmessage.append(tarball.name + ": Fix for " + cvestring.strip())
+            commitmessage.append("{}: Fix for {}"
+                                 .format(content.name, cvestring.strip()))
         else:
-            commitmessage.append(tarball.name + ": Autospec creation for version " +
+            commitmessage.append("{}: Autospec creation for version {}"
-                                 tarball.version)
+                                 .format(content.name, content.version))
     commitmessage.append("")
 
-    if have_cves:
+    # Only use Changelog if the giturl isn't defined as it is often
+    # duplicate content from the git log.
+    if content.giturl:
+        newsfiles = ["NEWS"]
+    else:
+        newsfiles = ["NEWS", "ChangeLog"]
+    for newsfile in newsfiles:
+        # parse news files for relevant version updates and cve fixes
+        newcommitmessage, newcves = process_NEWS(newsfile, config.old_version, content.name, content.version, config.download_path)
+        commitmessage.extend(newcommitmessage)
+        cves.update(newcves)
+
+    if cves:
+        # make the package security sensitive if a CVE was patched
+        config.config_opts['security_sensitive'] = True
+        config.rewrite_config_opts()
+        # append CVE fixes to end of commit message
         commitmessage.append("CVEs fixed in this build:")
-        commitmessage.extend(list(cves))
+        commitmessage.extend(sorted(list(cves)))
         commitmessage.append("")
 
-    process_NEWS("NEWS")
+    if keyinfo:
-    process_NEWS("ChangeLog")
+        commitmessage.append("Key imported:\n{}".format(keyinfo))
+
+    util.write_out(os.path.join(config.download_path, "commitmsg"),
+                   "\n".join(commitmessage) + "\n")
 
     print("Guessed commit message:")
-    print(commitmessage)
+    try:
-    with open(build.download_path + "/commitmsg", "w") as file:
+        print("\n".join(commitmessage))
-        file.write("\n".join(commitmessage) + "\n")
+    except Exception:
+        print("Can't print")

autospec/configure_blacklist

@@ -0,0 +1 @@
+whether double complex BLAS can be used

autospec/configure_whitelist

@@ -0,0 +1,680 @@
+acroread
+acroread4
+af77
+a french Unicode locale
+altivec.h
+altivec.h presence
+altivec.h usability
+ANSI C header files
+atan2pi
+atanpi
+a traditional french locale
+a traditional japanese locale
+a transitional chinese locale
+bison
+broken visibility attribute
+BSDgettimeofday
+/builddir/build/BUILD/gcc-build/./gcc/xgcc -B/builddir/build/BUILD/gcc-build/./gcc/ -B/usr/x86_64-generic-linux/bin/ -B/usr/x86_64-generic-linux/lib/ -isystem /usr/x86_64-generic-linux/include -isystem /usr/x86_64-generic-linux/sys-include    option to accept ISO C89
+c_asm.h
+c_asm.h presence
+c_asm.h usability
+cf77
+CFLocaleCopyCurrent
+CFPreferencesCopyAppValue
+com_err.h
+Configure miss: struct stat.st_atimespec
+cospi
+cospi in -lsunmath
+crt_externs.h
+crt_externs.h presence
+crt_externs.h usability
+dependency style of g++
+dependency style of gcc
+direct.h
+dl.h
+dl.h presence
+dl.h usability
+dlltool
+dlopen
+_doprnt
+doxygen
+dummy main to link with Fortran 77 libraries
+dynamic linker characteristics
+efc
+--enable-generated-files-in-srcdir
+--enable-version-specific-runtime-libs
+--enable-vtable-verify
+epcf90
+etags
+et/com_err.h
+evince
+external symbol _system_configuration
+extra flags to get ANSI library prototypes
+f77
+f90
+f95
+fdwalk
+_FILE_OFFSET_BITS value needed for large files
+firefox
+fl32
+flex
+floatingpoint.h
+floatingpoint.h presence
+floatingpoint.h usability
+fop
+fort
+fort77
+Fortran flag to compile .f90 files
+Fortran flag to compile .f95 files
+fprintf_unlocked
+frt
+g77
+galeon
+gccgo
+gcc option to accept ANSI C
+gcc option to accept ISO C89
+gcc option to accept ISO C99
+gcc option to enable C11 features
+gcj
+gethrtime
+getpass_r
+getsysinfo
+ggv
+gm4
+gnatbind
+gnatmake
+gnome-gv
+gnome-moz-remote
+gnum4
+gnutar
+gobject-introspection
+gpdf
+gtar
+gv
+how to run the C preprocessor
+HP-UX
+hrtime_t
+i686-pc-linux-gnu-ar
+i686-pc-linux-gnu-dlltool
+i686-pc-linux-gnu-gcc option to accept ISO C89
+i686-pc-linux-gnu-mt
+i686-pc-linux-gnu-objdump
+i686-pc-linux-gnu-ranlib
+i686-pc-linux-gnu-strip
+iconv in -liconv
+iconvlist
+ieeefp.h
+ieeefp.h presence
+ieeefp.h usability
+if /builddir/build/BUILD/gcc-build/./gcc/xgcc -B/builddir/build/BUILD/gcc-build/./gcc/ -B/usr/x86_64-generic-linux/bin/ -B/usr/x86_64-generic-linux/lib/ -isystem /usr/x86_64-generic-linux/include -isystem /usr/x86_64-generic-linux/sys-include    static flag -static works
+if /builddir/build/BUILD/gcc-build/./gcc/xgcc -B/builddir/build/BUILD/gcc-build/./gcc/ -B/usr/x86_64-generic-linux/bin/ -B/usr/x86_64-generic-linux/lib/ -isystem /usr/x86_64-generic-linux/include -isystem /usr/x86_64-generic-linux/sys-include    supports -fno-rtti -fno-exceptions
+ifc
+if closesocket can be linked
+if CloseSocket can be linked
+if closesocket might be used
+if CloseSocket might be used
+if compiler is clang
+if compiler is DEC/Compaq/HP C
+if compiler is HP-UX C
+if compiler is IBM C
+if compiler is Intel C
+if compiler is LCC
+if compiler is SGI MIPS C
+if compiler is SGI MIPSpro C
+if compiler is SunPro C
+if compiler is Tiny C
+if compiler is Watcom C
+if compiler supports -Werror=unknown-warning-option
+if compiler supports -Werror=unused-command-line-argument
+if c++ static flag -static works
+if ctime_r wants three arguments
+if f95 static flag -static works
+if gcc static flag -static works
+if gcc -std=gnu99 supports -fno-rtti -fno-exceptions
+if gcc supports -fno-rtti -fno-exceptions
+if gcc supports -Werror=unknown-warning-option
+if gcc supports -Werror=unused-command-line-argument
+if gethostbyaddr_r takes 5 args.
+if gethostbyaddr_r takes 7 args.
+if gethostbyaddr_r wants seven arguments
+if gethostbyname_r takes 3 args.
+if gethostbyname_r takes 5 args.
+if gethostbyname_r wants five arguments
+if getservbyport_r takes 4 args.
+if getservbyport_r takes 5 args.
+if gfortran PIC flag -fPIC works
+if gfortran static flag -static works
+if gfortran supports -c -o file.o
+if g++ static flag -static works
+if inet_ntoa_r can be linked
+if inet_ntoa_r might be used
+if ioctlsocket can be linked
+if IoctlSocket can be linked
+if ioctlsocket might be used
+if IoctlSocket might be used
+if : is a manifest tool
+if libcurl version >= 7.28.0
+if memrchr is prototyped
+if memrchr might be used
+if mkdir takes one argument
+if more special flags are required for pthreads
+if setsockopt SO_NONBLOCK is compilable
+if setsockopt SO_NONBLOCK might be used
+if sig_atomic_t is already defined as volatile
+if sigsetjmp can be linked
+if strcmpi can be linked
+if strcmpi might be used
+if strerror_r is glibc like
+if stricmp can be linked
+if stricmp might be used
+if strncmpi can be linked
+if strncmpi might be used
+if strnicmp can be linked
+if strnicmp might be used
+if /usr/bin/gcc static flag -static works
+if /usr/bin/gcc -std=gnu99 static flag -static works
+if /usr/bin/gcc -std=gnu99 supports -fno-rtti -fno-exceptions
+if /usr/bin/gcc supports -fno-rtti -fno-exceptions
+if /usr/local/bin/gcc supports -fno-rtti -fno-exceptions
+if <X11/extensions/XIproto.h> is needed for xReply
+intrinsics.h
+intrinsics.h presence
+intrinsics.h usability
+io.h
+io.h presence
+io.h usability
+is_selinux_enabled in -lselinux
+issetugid
+Kerberos 5
+kfmclient
+kghostview
+kpdf
+kpsewhich
+kqueue
+krb5-config
+_LARGEFILE_SOURCE value needed for large files
+lchmod
+ldap_ntlm_bind
+lex library
+lf95
+libpapi... checking for papiServiceCreate in -lpapi
+library containing bindtextdomain
+library containing clock_gettime
+library containing connect
+library containing getaddrinfo
+library containing gethostent
+library containing setsockopt
+library containing socket
+library containing strerror
+libutil.h
+libutil.h presence
+libutil.h usability
+lipo
+lp
+lpr
+mach_absolute_time
+machine/hal_sysinfo.h
+mach/mach_time.h
+mach/mach_time.h presence
+mach/mach_time.h usability
+main in -linet
+makeindex
+Minix Amsterdam compiler
+minix/config.h
+minix/config.h presence
+minix/config.h usability
+minix/sha2.h
+minix/sha2.h presence
+minix/sha2.h usability
+missing gethostname prototype
+_mm_free
+_mm_malloc
+mozilla
+mt
+multithread API to use
+nanotime
+native Win32
+net/errno.h
+net/errno.h presence
+net/errno.h usability
+_NSGetEnviron
+ocamlbuild
+okular
+open
+OpenLDAP
+opera
+pcre/pcre.h
+pcre/pcre.h presence
+pcre/pcre.h usability
+pdflatex
+pdftex
+perl5
+pgf77
+pgf90
+pgf95
+pghpf
+pown
+process.h
+pstat_getdynamic
+pthread_create
+read_real_time
+_rtc intrinsic
+__secure_getenv
+sepol_check_context in -lsepol
+__setfpucw
+_set_invalid_parameter_handler
+setmode
+setproctitle
+shl_load
+shl_load in -ldld
+sin
+sinpi
+_snprintf
+socket in -lsocket
+some Win32 platform
+spawnve
+special C compiler options needed for large files
+stricmp
+strlcat
+strlcpy
+strnicmp
+struct sockaddr_in.sin_len
+struct sockaddr.sa_len
+struct sockaddr_un.sun_len
+struct stat.st_birthtimensec
+struct stat.st_birthtimespec.tv_nsec
+struct stat.st_birthtim.tv_nsec
+struct tm.__tm_gmtoff
+SunLDAP
+sunmath.h
+sunmath.h presence
+sunmath.h usability
+sys/endian.h
+sys/endian.h presence
+sys/endian.h usability
+sys/filio.h
+sys/filio.h presence
+sys/filio.h usability
+sys/limits.h
+sys/loadavg.h
+sys/loadavg.h presence
+sys/loadavg.h usability
+sys/mkdev.h
+sys/mkdev.h presence
+sys/mkdev.h usability
+sys/pstat.h
+sysroot
+sys/sockio.h
+sys/sockio.h presence
+sys/sockio.h usability
+sys/systeminfo.h
+sys/systeminfo.h presence
+sys/systeminfo.h usability
+table
+tanpi
+tex
+texi2any
+texi2dvi
+the BeOS
+the pthreads library -llthread
+the pthreads library -lpthreads
+time_base_to_time
+/usr/bin/gcc option to accept ISO C89
+/usr/local/bin/gcc option to accept ISO C89
+util.h
+util.h presence
+util.h usability
+valgrind.h
+valgrind.h presence
+vasnprintf
+vfork.h
+vfork.h presence
+vfork.h usability
+_vsnprintf
+whether alloca needs Cray hooks
+whether byte ordering is bigendian
+whether bzip2 support needs to be compiled
+whether  can compile ObjC++
+whether canonicalize_file_name must be declared
+whether char is unsigned
+whether __clang__ is declared
+whether closedir returns void
+whether compiler driver understands Ada
+whether cosq is declared
+whether C runtime needs -D__NO_MATH_INLINES
+whether dirfd is a macro
+whether f95 appends extra underscores to external names
+whether fprintf_unlocked is declared
+whether fpurge is declared without a macro
+whether g++ can compile ObjC++
+whether gcc accepts -g
+whether GCC assembler is compatible for ARM assembly implementations
+whether gcc needs -traditional
+whether gets is declared without a macro
+whether gettimeofday clobbers localtime buffer
+whether g++  supports C++11 features by default
+whether included gettext is requested
+whether integer division by zero raises SIGFPE
+whether __INTEL_COMPILER is declared
+whether // is distinct from /
+whether KERN_USRSTACK sysctl is supported
+whether -lc should be explicitly linked in
+whether long double and double are the same
+whether mktime sets errno
+whether mktime works correctly outside 1902-2037
+whether PCRE support needs to be compiled
+whether pthreads work with -kthread
+whether pthreads work with -Kthread
+whether pthreads work without any flags
+whether putenv("FOO=") can unset an environment variable
+whether sinq is declared
+whether _snprintf is declared
+whether _snwprintf is declared
+whether stat accepts an empty string
+whether stat file-mode macros are broken
+whether stricmp is declared
+whether strlcpy is declared
+whether strnicmp is declared
+whether __SUNPRO_C is declared
+whether system header files limit the line length
+whether termios.h defines TIOCGWINSZ
+whether the compiler supports -xldscope=hidden
+whether the inttypes.h PRIxNN macros are broken
+whether the thread-local storage support is from emutls
+whether to build gtk-doc documentation
+whether to build shared libraries
+whether to build static libraries
+whether to enable maintainer-specific portions of Makefiles
+whether to install libiberty headers and static library
+whether to use ARM IWMMXT intrinsics
+whether to use ARM NEON assembler
+whether to use ARM SIMD assembler
+whether to use Loongson MMI assembler
+whether to use MIPS DSPr2 assembler
+whether to use VMX/Altivec intrinsics
+Whether to write dependencies into .pc files
+whether we are cross compiling
+whether we are using the GNU Objective C compiler
+whether we can compute ObjC Make dependencies
+whether _XOPEN_SOURCE should be defined
+whether yytext is a pointer
+whether zlib support needs to be compiled
+Win32
+Win32 threads
+windows.h
+windows.h presence
+windows.h usability
+x86_64-generic-linux-ar
+x86_64-generic-linux-as
+x86_64-generic-linux-cc
+x86_64-generic-linux-dlltool
+x86_64-generic-linux-gccgo
+x86_64-generic-linux-gcj
+x86_64-generic-linux-gnu-aCC
+x86_64-generic-linux-gnu-af77
+x86_64-generic-linux-gnu-ar
+x86_64-generic-linux-gnu-c++
+x86_64-generic-linux-gnu-cc
+x86_64-generic-linux-gnu-cc++
+x86_64-generic-linux-gnu-CC
+x86_64-generic-linux-gnu-cf77
+x86_64-generic-linux-gnu-cl
+x86_64-generic-linux-gnu-cl.exe
+x86_64-generic-linux-gnu-cxx
+x86_64-generic-linux-gnu-dlltool
+x86_64-generic-linux-gnu-efc
+x86_64-generic-linux-gnu-epcf90
+x86_64-generic-linux-gnu-f77
+x86_64-generic-linux-gnu-f90
+x86_64-generic-linux-gnu-f95
+x86_64-generic-linux-gnu-FCC
+x86_64-generic-linux-gnu-fl32
+x86_64-generic-linux-gnu-fort
+x86_64-generic-linux-gnu-fort77
+x86_64-generic-linux-gnu-frt
+x86_64-generic-linux-gnu-ftn
+x86_64-generic-linux-gnu-g++
+x86_64-generic-linux-gnu-g77
+x86_64-generic-linux-gnu-g95
+x86_64-generic-linux-gnu-gcc
+x86_64-generic-linux-gnu-gfortran
+x86_64-generic-linux-gnu-gpp
+x86_64-generic-linux-gnu-ifc
+x86_64-generic-linux-gnu-ifort
+x86_64-generic-linux-gnu-KCC
+x86_64-generic-linux-gnu-lf95
+x86_64-generic-linux-gnu-lib
+x86_64-generic-linux-gnu-link
+x86_64-generic-linux-gnu-mt
+x86_64-generic-linux-gnu-nagfor
+x86_64-generic-linux-gnu-nm
+x86_64-generic-linux-gnu-objc
+x86_64-generic-linux-gnu-objcc
+x86_64-generic-linux-gnu-objdump
+x86_64-generic-linux-gnu-pgf77
+x86_64-generic-linux-gnu-pgf90
+x86_64-generic-linux-gnu-pgf95
+x86_64-generic-linux-gnu-pgfortran
+x86_64-generic-linux-gnu-pghpf
+x86_64-generic-linux-gnu-ranlib
+x86_64-generic-linux-gnu-RCC
+x86_64-generic-linux-gnu-strings
+x86_64-generic-linux-gnu-strip
+x86_64-generic-linux-gnu-windres
+x86_64-generic-linux-gnu-xlC
+x86_64-generic-linux-gnu-xlC_r
+x86_64-generic-linux-gnu-xlf
+x86_64-generic-linux-gnu-xlf90
+x86_64-generic-linux-gnu-xlf95
+x86_64-generic-linux-ld
+x86_64-generic-linux-lipo
+x86_64-generic-linux-nm
+x86_64-generic-linux-objcopy
+x86_64-generic-linux-objdump
+x86_64-generic-linux-ranlib
+x86_64-generic-linux-readelf
+x86_64-generic-linux-strip
+x86_64-generic-linux-windmc
+x86_64-generic-linux-windres
+x86_64-koji-linux-gnu-aCC
+x86_64-koji-linux-gnu-af77
+x86_64-koji-linux-gnu-ar
+x86_64-koji-linux-gnu-c++
+x86_64-koji-linux-gnu-cc++
+x86_64-koji-linux-gnu-CC
+x86_64-koji-linux-gnu-cl
+x86_64-koji-linux-gnu-cxx
+x86_64-koji-linux-gnu-dlltool
+x86_64-koji-linux-gnu-efc
+x86_64-koji-linux-gnu-epcf90
+x86_64-koji-linux-gnu-f77
+x86_64-koji-linux-gnu-f90
+x86_64-koji-linux-gnu-f95
+x86_64-koji-linux-gnu-FCC
+x86_64-koji-linux-gnu-fl32
+x86_64-koji-linux-gnu-fort
+x86_64-koji-linux-gnu-fort77
+x86_64-koji-linux-gnu-frt
+x86_64-koji-linux-gnu-g++
+x86_64-koji-linux-gnu-g77
+x86_64-koji-linux-gnu-gfortran
+x86_64-koji-linux-gnu-gpp
+x86_64-koji-linux-gnu-ifc
+x86_64-koji-linux-gnu-KCC
+x86_64-koji-linux-gnu-lf95
+x86_64-koji-linux-gnu-mt
+x86_64-koji-linux-gnu-objdump
+x86_64-koji-linux-gnu-pgf77
+x86_64-koji-linux-gnu-pgf90
+x86_64-koji-linux-gnu-pgf95
+x86_64-koji-linux-gnu-ranlib
+x86_64-koji-linux-gnu-RCC
+x86_64-koji-linux-gnu-strings
+x86_64-koji-linux-gnu-strip
+x86_64-koji-linux-gnu-windres
+x86_64-koji-linux-gnu-xlC
+x86_64-koji-linux-gnu-xlC_r
+x86_64-koji-linux-gnu-xlf
+x86_64-koji-linux-gnu-xlf90
+x86_64-koji-linux-gnu-xlf95
+x86_64-pc-linux-gnu-aCC
+x86_64-pc-linux-gnu-af77
+x86_64-pc-linux-gnu-ar
+x86_64-pc-linux-gnu-c++
+x86_64-pc-linux-gnu-cc++
+x86_64-pc-linux-gnu-CC
+x86_64-pc-linux-gnu-cl
+x86_64-pc-linux-gnu-cl.exe
+x86_64-pc-linux-gnu-cxx
+x86_64-pc-linux-gnu-dlltool
+x86_64-pc-linux-gnu-efc
+x86_64-pc-linux-gnu-epcf90
+x86_64-pc-linux-gnu-f77
+x86_64-pc-linux-gnu-f90
+x86_64-pc-linux-gnu-f95
+x86_64-pc-linux-gnu-FCC
+x86_64-pc-linux-gnu-fl32
+x86_64-pc-linux-gnu-fort
+x86_64-pc-linux-gnu-fort77
+x86_64-pc-linux-gnu-frt
+x86_64-pc-linux-gnu-g++
+x86_64-pc-linux-gnu-g77
+x86_64-pc-linux-gnu-gcc
+x86_64-pc-linux-gnu-gfortran
+x86_64-pc-linux-gnu-gpp
+x86_64-pc-linux-gnu-ifc
+x86_64-pc-linux-gnu-KCC
+x86_64-pc-linux-gnu-lf95
+x86_64-pc-linux-gnu-mt
+x86_64-pc-linux-gnu-objdump
+x86_64-pc-linux-gnu-pgf77
+x86_64-pc-linux-gnu-pgf90
+x86_64-pc-linux-gnu-pgf95
+x86_64-pc-linux-gnu-pkg-config
+x86_64-pc-linux-gnu-ranlib
+x86_64-pc-linux-gnu-RCC
+x86_64-pc-linux-gnu-strip
+x86_64-pc-linux-gnu-xlC
+x86_64-pc-linux-gnu-xlC_r
+x86_64-pc-linux-gnu-xlf
+x86_64-pc-linux-gnu-xlf90
+x86_64-pc-linux-gnu-xlf95
+x86_64-unknown-linux-gnu-ar
+x86_64-unknown-linux-gnu-dlltool
+x86_64-unknown-linux-gnu-lib
+x86_64-unknown-linux-gnu-link
+x86_64-unknown-linux-gnu-mt
+x86_64-unknown-linux-gnu-objdump
+x86_64-unknown-linux-gnu-ranlib
+x86_64-unknown-linux-gnu-strip
+xdg-open
+xlf
+xlf90
+xlf95
+xmkmf
+X.Org SGML entities >= 1.8
+xpdf
+ansidecl.h usability
+ansidecl.h presence
+ansidecl.h
+nan.h usability
+nan.h presence
+nan.h
+fp_class.h usability
+fp_class.h presence
+fp_class.h
+isnand
+fp_class
+class
+fpclass
+_stat
+ansidecl.h usability
+ansidecl.h presence
+ansidecl.h
+nan.h usability
+nan.h presence
+nan.h
+fp_class.h usability
+fp_class.h presence
+fp_class.h
+isnand
+fp_class
+class
+fpclass
+_stat
+whether g++ supports C++11 features by default
+tsol/label.h usability
+tsol/label.h presence
+tsol/label.h
+is_system_labeled
+launchd
+getlocalename_l
+net/if_dl.h usability
+net/if_dl.h presence
+net/if_dl.h
+sys/disklabel.h usability
+sys/disklabel.h presence
+sys/disklabel.h
+sys/disk.h usability
+sys/disk.h presence
+sys/disk.h
+whether llseek is declared
+whether struct stat has a st_flags field
+chflags
+fadvise64
+getmntinfo
+optreset
+sem_init
+whether we can link with -static
+if building for some Win32 platform
+more warnings
+x86_64-generic-linux-gnu-makeindex
+x86_64-generic-linux-gnu-pdflatex
+x86_64-generic-linux-gnu-egrep
+x86_64-generic-linux-gnu-dvips
+x86_64-generic-linux-gnu-latex
+x86_64-generic-linux-gnu-dot
+x86_64-generic-linux-gnu-perl
+x86_64-generic-linux-gnu-doxygen
+latex
+dvips
+whether more special flags are required for pthreads
+Configure miss: whether to build with gcov testing
+if compiler needs -Werror to reject unknown flags
+pow
+struct stat.st_mtimensec
+struct statvfs.f_basetype
+struct statvfs.f_fstypename
+sys/audioio.h presence
+valgrind/valgrind.h presence
+valgrind/valgrind.h usability
+whether C compiler handles -Werror -Wunknown-warning-option
+whether self tests are run under valgrind
+arc4random_buf
+sys/ucred.h
+sys/utime.h
+valgrind/valgrind.h
+struct statfs.f_fstypename
+whether __argv is declared
+whether <limits.h> defines MIN and MAX
+whether C compiler accepts -Werror=unknown-warning-option
+whether gcc is Clang
+whether more special flags are required for pthreads
+whether the C locale is free of encoding errors
+winsock2.h
+getpeereid
+getpeerucred
+whether wint_t is too small
+xlocale.h usability
+whether the linker accepts -Wl,-fatal_warnings
+whether to build with code coverage support
+xlocale.h presence
+getexecname
+getprogname
+whether stdint.h predates C++11
+xlocale.h
+i686-generic-linux-gnu-pkg-config
+i686-generic-linux-gnu-objdump
+i686-generic-linux-gnu-dlltool
+i686-generic-linux-gnu-mt
+i686-generic-linux-gnu-strip

autospec/docs.py

@@ -1,48 +0,0 @@
-#!/bin/true
-#
-# docs.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import os
-import shutil
-import sys
-
-transforms = {
-    'changes': 'ChangeLog',
-    'changelog.txt': 'ChangeLog',
-    'changelog': 'ChangeLog',
-    'changes.rst': 'ChangeLog',
-    'changes.txt': 'ChangeLog',
-    'news': 'NEWS'
-}
-interests = transforms.keys()
-
-
-def scan_for_changes(download_path, dir):
-    found = []
-    for dirpath, dirnames, files in os.walk(dir, topdown=False):
-
-        hits = [x for x in files if x.lower() in interests and x.lower() not in found]
-        for item in hits:
-            source = os.path.join(dirpath, item)
-            target = os.path.join(download_path, transforms[item.lower()])
-            try:
-                shutil.copy(source, target)
-            except Exception as e:
-                print("Error copying file: %s", e)
-                sys.exit(1)
-            found.append(item)

autospec/download.py

@@ -0,0 +1,79 @@
+#!/usr/bin/true
+#
+# download.py - part of autospec
+# Copyright (C) 2018 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+from io import BytesIO
+
+import pycurl
+from util import print_fatal
+
+
+def do_curl(url, dest=None, post=None, is_fatal=False):
+    """
+    Perform a curl operation for `url`.
+
+    If `post` is set, a POST is performed for `url` with fields taken from the
+    specified value. Otherwise a GET is performed for `url`. If `dest` is set,
+    the curl response (if successful) is written to the specified path and the
+    path is returned. Otherwise a successful response is returned as a BytesIO
+    object. If `is_fatal` is `True` (`False` is the default), a GET failure,
+    POST failure, or a failure to write to the path specified for `dest`
+    results in the program exiting with an error. Otherwise, `None` is returned
+    for any of those error conditions.
+    """
+    c = pycurl.Curl()
+    c.setopt(c.URL, url)
+    if post:
+        c.setopt(c.POSTFIELDS, post)
+    c.setopt(c.FOLLOWLOCATION, True)
+    c.setopt(c.FAILONERROR, True)
+    c.setopt(c.CONNECTTIMEOUT, 10)
+    c.setopt(c.TIMEOUT, 600)
+    c.setopt(c.LOW_SPEED_LIMIT, 1)
+    c.setopt(c.LOW_SPEED_TIME, 10)
+    buf = BytesIO()
+    c.setopt(c.WRITEDATA, buf)
+    try:
+        c.perform()
+    except pycurl.error as e:
+        if is_fatal:
+            print_fatal("Unable to fetch {}: {}".format(url, e))
+            sys.exit(1)
+        return None
+    finally:
+        c.close()
+
+    # write to dest if specified
+    if dest:
+        try:
+            with open(dest, 'wb') as fp:
+                fp.write(buf.getvalue())
+        except IOError as e:
+            if os.path.exists(dest):
+                os.unlink(dest)
+            if is_fatal:
+                print_fatal("Unable to write to {}: {}".format(dest, e))
+                sys.exit(1)
+            return None
+
+    if dest:
+        return dest
+    else:
+        return buf

autospec/files.py

@@ -19,382 +19,436 @@
 # %files section management
 #
 
-import build
+import os
-from collections import OrderedDict
-import lang
 import re
-import tarball
+from collections import OrderedDict
-import buildreq
-import config
-# todo package splits
 
-# per sub-package file list for spec purposes
+import util
-packages = OrderedDict()
-main_requires = OrderedDict()
-
-# global file list to weed out dupes
-files = []
-files_blacklist = []
-excludes = []
-extras = []
-setuid = []
-attrs = {}
-
-newfiles_printed = 0
-
-#
-# Do we need ALL include files in a dev package, even if they're not in /usr/include?
-# Yes in the general case, but for example for R packages,
-# the answer is No.
-want_dev_split = 1
 
 
-def push_main_requires(package):
+class FileManager(object):
-    global main_requires
+    """Class to handle spec file %files section management."""
-    if (package not in main_requires):
-        main_requires[package] = set()
 
+    def __init__(self, config, package):
+        """Set defaults for FileManager."""
+        self.config = config
+        self.package = package
+        self.packages = OrderedDict()  # per sub-package file list for spec purposes
+        self.files = set()  # global file set to weed out dupes
+        self.files_blacklist = set()
+        self.excludes = []
+        self.manual_excludes = []
+        self.file_maps = {}  # Filename-to-package mapping
+        self.setuid = []
+        self.attrs = {}
+        self.locales = []
+        self.newfiles_printed = False
+        # Do we need ALL include files in a dev package, even if they're not in
+        # /usr/include?  Yes in the general case, but for example for R
+        # packages, the answer is No.
+        self.want_dev_split = True
+        self.has_banned = False
 
-def push_package_file(filename, package="main"):
+    @staticmethod
-    global packages
+    def banned_path(path):
-    global newfiles_printed
+        """Check if the path is either banned or in a banned subdirectory."""
+        banned_paths = [r"/etc.*",
+                        r"/opt.*",
+                        r"/usr/local.*",
+                        r"/usr/etc.*",
+                        r"/usr/src.*",
+                        r"/var.*"]
+        for bpath in banned_paths:
+            if re.search(r"^(/V3|/V4|/VA)?" + bpath, path):
+                return True
+        return False
 
-    if (package not in packages):
+    def push_package_file(self, filename, package="main"):
-        packages[package] = set()
+        """Add found %file and indicate to build module that we must restart the build."""
-    packages[package].add(filename)
+        if package not in self.packages:
-    build.must_restart = build.must_restart + 1
+            self.packages[package] = set()
-    if newfiles_printed == 0:
-        print("  New %files content found")
-        newfiles_printed = 1
 
+        if FileManager.banned_path(filename):
+            util.print_warning(f"  Content {filename} found in banned path, skipping")
+            self.has_banned = True
+            return
 
-def file_pat_match(filename, pattern, package, replacement="", prefix=""):
+        # prepend the %attr macro if file defined in 'attrs' control file
-    if replacement == "":
+        if filename in self.attrs:
-        replacement = prefix + filename
+            mod = self.attrs[filename][0]
+            u = self.attrs[filename][1]
+            g = self.attrs[filename][2]
+            filename = "%attr({0},{1},{2}) {3}".format(mod, u, g, filename)
+        self.packages[package].add(filename)
+        self.package.file_restart += 1
+        if not self.newfiles_printed:
+            print("  New %files content found")
+            self.newfiles_printed = True
 
-    pat = re.compile(pattern)
+    def compat_exclude(self, filename):
-    match = pat.search(filename)
+        """Exclude non-library files if the package is for compatability."""
-    if match:
+        if not self.config.config_opts.get("compat"):
-        if filename in excludes:
+            return False
-            push_package_file("%exclude " + filename, package)
+
+        patterns = [
+            r"/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so",
+            r"/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so",
+            r"/usr/lib64/haswell/[a-zA-Z0-9\.\_\-\+]*\.so\.",
+            r"/usr/share/package-licenses/"]
+
+        exclude = True
+        for pat in patterns:
+            pat = re.compile(r"^(/V3|/V4|/VA)?" + pat)
+            if pat.search(filename):
+                exclude = False
+                break
+
+        return exclude
+
+    def file_pat_match(self, filename, pattern, package, replacement=""):
+        """Search for pattern in filename.
+
+        Attempt to find pattern in filename, if pattern matches push package file.
+        If that file is also in the excludes list, don't push the file.
+        Returns True if a file was pushed, False otherwise.
+        """
+        if not replacement or self.config.config_opts.get("no_glob"):
+            replacement = filename
+
+        # compat files should always be excluded
+        if self.compat_exclude(filename):
+            self.excludes.append(filename)
             return True
-        push_package_file(replacement, package)
+
-        return True
+        # All patterns at this time and should always be prefixed by '^'
-    else:
+        # but just in case add the following to strip just the '^'
+        pattern = pattern if not pattern.startswith('^') else pattern[1:]
+        pat = re.compile(r"^(/V3|/V4|/VA)?" + pattern)
+        match = pat.search(filename)
+        if match:
+            if len(match.groups()) > 0 and match.groups()[0] in ['/V3', '/V4', '/VA']:
+                norm_filename = filename.removeprefix(match.groups()[0])
+                if replacement != filename:
+                    replacement = match.groups()[0] + replacement
+            else:
+                norm_filename = filename
+            if norm_filename in self.excludes:
+                return True
+
+            self.push_package_file(replacement, package)
+            return True
+
         return False
 
+    def file_is_locale(self, filename):
+        """If a file is a locale, appends to self.locales and returns True, returns False otherwise."""
+        pat = re.compile(r"^/usr/share/locale/.*/(.*)\.mo")
+        match = pat.search(filename)
+        if match:
+            lang = match.group(1)
+            if lang not in self.locales:
+                self.locales.append(lang)
+                print("  New locale:", lang)
+                self.package.must_restart += 1
+                if "locales" not in self.packages:
+                    self.packages["locales"] = set()
 
-def file_is_locale(filename):
+            return True
-    pat = re.compile("^/usr/share/locale/.*/(.*)\.mo")
-    match = pat.search(filename)
-    if match:
-        l = match.group(1)
-        lang.add_lang(l)
-        return True
-    else:
-        return False
-
-
-def push_file(filename):
-    global files
-    global files_blacklist
-    global extras
-    global setuid
-    global attrs
-    for file in files:
-        if file == filename:
-            return 0
-
-    if filename in files_blacklist:
-        return 0
-    files.append(filename)
-
-    if file_is_locale(filename):
-        return
-
-    # extras
-    if filename in extras:
-        push_package_file(filename, "extras")
-        excludes.append(filename)
-
-    if filename in setuid:
-        fn = "%attr(4755, root, root) " + filename
-        push_package_file(fn, "setuid")
-        excludes.append(filename)
-
-    if filename in attrs:
-        fn = "%s(%s,%s,%s) %s" % (attrs[filename][0], attrs[filename][1], attrs[filename][2],
-                                  attrs[filename][3], filename)
-        push_package_file(fn, "attr")
-        excludes.append(filename)
-
-    if file_pat_match(filename, r"^/usr/share/omf", "main", "/usr/share/omf/*"):
-        return
-
-    if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib", "/usr/lib/*.so.*"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib", "/usr/lib64/*.so.*"):
-        return
-    if file_pat_match(filename, r"^/usr/libexec/", "bin"):
-        return
-    if file_pat_match(filename, r"^/usr/bin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/usr/sbin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/sbin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/bin/", "bin"):
-        return
-    if file_pat_match(filename, r"^/bin/", "bin"):
-        return
-
-    if file_pat_match(filename, r"^/usr/lib/python.*/", "python", "/usr/lib/python*/*"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/python.*/", "python", "/usr/lib64/python*/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/gir-1\.0/.*\.gir\$", "data", "/usr/share/gir-1.0/*.gir"):
-        return
-    if file_pat_match(filename, r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/girepository-1\.0/.*\.typelib\$", "data", "/usr/share/girepository-1.0/*.typelib"):
-        return
-
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hxx", "dev", "/usr/include/*.hxx"):
-        return
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hpp", "dev", "/usr/include/*.hpp"):
-        return
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h\+\+", "dev", "/usr/include/*.h\+\+"):
-        return
-    if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h", "dev", "/usr/include/*.h"):
-        return
-    if file_pat_match(filename, r"^/usr/include/", "dev"):
-        return
-    if want_dev_split > 0 and file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev", "/usr/lib/*.so"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev", "/usr/lib64/*.so"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib/*.a"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib64/*.a"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev", "/usr/lib/pkgconfig/*.pc"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev", "/usr/lib64/pkgconfig/*.pc"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal/*.ac"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal/*.m4"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"):
-        return
-    if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"):
-        return
-
-    if file_pat_match(filename, r"^/usr/share/doc/" + tarball.name + "/", "doc", "%doc /usr/share/doc/" + tarball.name + "/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/gtk-doc/html", "doc"):
-        return
-    if file_pat_match(filename, r"^/usr/share/info/", "doc", "%doc /usr/share/info/*"):
-        return
-
-    if file_pat_match(filename, r"^/usr/share/man/man0", "doc", "%doc /usr/share/man/man0/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man1", "doc", "%doc /usr/share/man/man1/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man2", "doc", "%doc /usr/share/man/man2/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man3", "doc", "%doc /usr/share/man/man3/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man4", "doc", "%doc /usr/share/man/man4/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man5", "doc", "%doc /usr/share/man/man5/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man6", "doc", "%doc /usr/share/man/man6/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man7", "doc", "%doc /usr/share/man/man7/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man8", "doc", "%doc /usr/share/man/man8/*"):
-        return
-    if file_pat_match(filename, r"^/usr/share/man/man9", "doc", "%doc /usr/share/man/man9/*"):
-        return
-
-    if file_pat_match(filename, r"^/etc/systemd/system/.*\.wants/", "active-units"):
-        return
-
-    # now a set of catch-all rules
-    if file_pat_match(filename, r"^/etc/", "config", "", "%config "):
-        return
-    if file_pat_match(filename, r"^/usr/etc/", "config", "", "%config "):
-        return
-    if file_pat_match(filename, r"^/lib/systemd", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/systemd", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/udev/rules.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/modules-load.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/tmpfiles.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/sysusers.d", "config"):
-        return
-    if file_pat_match(filename, r"^/usr/lib/sysctl.d", "config"):
-        return
-
-    if file_pat_match(filename, r"^/usr/share/", "data"):
-        return
-
-    # finally move any dynamically loadable plugins (not
-    # perl/python/ruby/etc.. extensions) into lib package
-    if file_pat_match(filename, r"^/usr/lib/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"):
-        return
-    if file_pat_match(filename, r"^/usr/lib64/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"):
-        return
-
-# locale data gets picked up via find_lang
-    if file_pat_match(filename, r"^/usr/share/locale/", "ignore"):
-        return
-
-    if filename in excludes:
-        push_package_file("%exclude " + filename)
-        return
-
-    push_package_file(filename)
-
-
-def remove_file(file):
-    global files
-    global packages
-    global files_blacklist
-    hit = False
-
-    if file in files:
-        files.remove(file)
-        print("File no longer present: %s" % file)
-        hit = True
-    for pkg in packages:
-        if file in packages[pkg]:
-            packages[pkg].remove(file)
-            print("File no longer present in %s: %s" % (pkg, file))
-            hit = True
-    if hit:
-        if file not in files_blacklist:
-            files_blacklist.append(file)
-        build.must_restart = build.must_restart + 1
-
-
-def write_files_header(file):
-    global files
-    global packages
-
-    groups = dict()
-    groups["dev"] = "Development"
-    groups["bin"] = "Binaries"
-    groups["lib"] = "Libraries"
-    groups["doc"] = "Documentation"
-    groups["data"] = "Data"
-
-    deps = dict()
-
-    deps["dev"] = ["lib", "bin", "data"]
-    deps["bin"] = ["data", "config", "setuid", "attr"]
-    deps["lib"] = ["data", "config"]
-
-    provides = dict()
-    provides["dev"] = ["devel"]
-
-    for pkg in sorted(packages):
-        if pkg == "ignore":
-            continue
-        if pkg == "main":
-            continue
-
-        file.write("\n%package " + pkg + "\n")
-        file.write("Summary: " + pkg +
-                   " components for the " + tarball.name + " package.\n")
-        if pkg in groups:
-            file.write("Group: " + groups[pkg] + "\n")
         else:
-            file.write("Group: Default\n")
+            return False
-        for dep in deps.get(pkg, []):
-            if dep in packages:
-                file.write("Requires: " + tarball.name + "-" + dep + "\n")
-        for prov in provides.get(pkg, []):
-            file.write("Provides: " + tarball.name + "-" + prov + "\n")
-        if pkg == "python":
-            if tarball.name != tarball.name.lower():
-                file.write("Provides: %s-python\n" % tarball.name.lower())
-            for req in sorted(buildreq.pythonreqs):
-                if req in buildreq.buildreqs:
-                    file.write("Requires: " + req + "\n")
-        file.write("\n")
-        file.write("%description " + pkg + "\n")
-        file.write(pkg + " components for the " + tarball.name + " package.\n")
-        file.write("\n")
 
+    def _clean_dirs(self, root, files):
+        """Do the work to remove the directories from the files list."""
+        res = set()
+        removed = False
 
-def write_files(file):
+        directive_re = re.compile(r"(%\w+(\([^\)]*\))?\s+)(.*)")
-    global files
+        for f in files:
-    global packages
+            # skip the files with directives at the beginning, including %doc
+            # and %dir directives.
+            # autospec does not currently support adding empty directories to
+            # the file list by prefixing "%dir". Regardless, skip these entries
+            # because if they exist at this point it is intentional (i.e.
+            # support was added).
+            if directive_re.match(f):
+                res.add(f)
+                continue
 
-    file.write("\n%files\n")
+            path = os.path.join(root, f.lstrip("/"))
-    file.write("%defattr(-,root,root,-)\n")
+            if os.path.isdir(path) and not os.path.islink(path):
-    if "main" in packages:
+                util.print_warning("Removing directory {} from file list".format(f))
-        for filename in sorted(packages["main"]):
+                self.files_blacklist.add(f)
-            file.write(filename + "\n")
+                removed = True
+            else:
+                res.add(f)
 
-    for pkg in sorted(packages):
+        return (res, removed)
-        if pkg == "ignore":
-            continue
-        if pkg == "main":
-            continue
-        if pkg == "locales":
-            continue
-        file.write("\n%files " + pkg + "\n")
-        file.write("%defattr(-,root,root,-)\n")
-        for filename in sorted(packages[pkg]):
-            file.write(filename + "\n")
 
+    def clean_directories(self, root):
+        """Remove directories from file list."""
+        removed = False
+        for pkg in self.packages:
+            self.packages[pkg], _rem = self._clean_dirs(root, self.packages[pkg])
+            if _rem:
+                removed = True
 
-def write_scriplets(file):
+        return removed
-    global files
-    global packages
 
-    for pkg in sorted(packages):
+    def globlike_match(self, filename, match_name):
-        if pkg == "ignore":
+        """Compare the filename to the match_name in a way that simulates the shell glob '*'."""
-            continue
+        fsplit = filename.split('/')
-        if pkg == "main":
+        if len(fsplit) != len(match_name):
-            continue
+            return False
-        if pkg == "locales":
+        match = True
-            continue
+        for fpart, mpart in zip(fsplit, match_name):
-        for script in ["post", "pre"]:
+            if fpart != mpart:
-            content = config.read_conf_file("%s.%s" % (pkg, script))
+                if '*' not in mpart:
-            if content:
+                    match = False
-                file.write("\n%{0} {1}\n".format(script, pkg))
+                    break
-                file.writelines(content)
+                if len(mpart) > len(fpart) + 1:
+                    match = False
+                    break
+                mpl, mpr = mpart.split('*')
+                try:
+                    if fpart.index(mpl) != 0:
+                        match = False
+                        break
+                    if fpart.rindex(mpr) != len(fpart) - len(mpr):
+                        match = False
+                        break
+                except ValueError:
+                    match = False
+                    break
+        return match
 
+    def push_file(self, filename, pkg_name):
+        """Perform a number of checks against the filename and push the filename if appropriate."""
+        if filename in self.files or filename in self.files_blacklist:
+            return
 
-def write_main_subpackage_requires(file):
+        self.files.add(filename)
-    global files
+        if self.file_is_locale(filename):
-    global packages
+            return
-    global main_requires
 
-    for pkg in packages:
+        # Explicit file packaging
-        if pkg == "ignore":
+        for k, v in self.file_maps.items():
-            continue
+            for match_name in v['files']:
-        if pkg == "main":
+                match = re.search(r"^/(V3|V4|VA)", filename)
-            continue
+                norm_filename = filename if not match else filename.removeprefix(match.group())
-        if pkg == "dev":
+                if isinstance(match_name, str):
-            continue
+                    if norm_filename == match_name:
-        if pkg == "active-units":
+                        self.push_package_file(filename, k)
-            continue
+                        return
-        if pkg == "extras":
+                elif len('/'.join(match_name)) <= (len(norm_filename) + 1):
-            continue
+                    # the match name may be 1 longer due to a glob
-        file.write("Requires: " + tarball.name + "-" + pkg + "\n")
+                    # being able to match an empty string
+                    if self.globlike_match(norm_filename, match_name):
+                        path_prefix = '/' if not match else match.group()
+                        self.push_package_file(os.path.join(path_prefix, *match_name), k)
+                        return
 
-    for pkg in main_requires:
+        if filename in self.setuid:
-        file.write("Requires: " + pkg + "\n")
+            if filename in self.attrs:
+                mod = self.attrs[filename][0]
+                u = self.attrs[filename][1]
+                g = self.attrs[filename][2]
+                newfn = "%attr({0},{1},{2}) {3}".format(mod, u, g, filename)
+            else:
+                newfn = "%attr(4755, root, root) " + filename
+            self.push_package_file(newfn, "setuid")
+            return
+
+        # autostart
+        part = re.compile(r"^/usr/lib/systemd/system/.+\.target\.wants/.+")
+        if part.search(filename) and 'update-triggers.target.wants' not in filename:
+            if filename not in self.excludes:
+                self.push_package_file(filename, "autostart")
+                self.push_package_file("%exclude " + filename, "services")
+                return
+
+        if self.want_dev_split and self.file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"):
+            return
+
+        # Exclude Windows executables and DLLs unless otherwise configured
+        # Can't just skip them because they could be swept up in a python lib wildcard, for example
+        if re.search(r"[^/]+\.(exe|dll)$", filename):
+            if self.config.config_opts.get('allow_exe'):
+                util.print_warning("Allowing {} because allow_exe is true".format(filename))
+            else:
+                util.print_warning("Blocking {} because allow_exe is false".format(filename))
+                self.excludes.append(filename)
+                return
+
+        # if configured to do so, add .so files to the lib package instead of
+        # the dev package. THis is useful for packages with a plugin
+        # architecture like elfutils and mesa.
+        so_dest = 'lib' if self.config.config_opts.get('so_to_lib') else 'dev'
+        so_dest_ompi = 'openmpi' if self.config.config_opts.get('so_to_lib') else 'dev'
+
+        patterns = [
+            # Patterns for matching files, format is a tuple as follows:
+            # (<raw pattern>, <package>, <optional replacement>, <optional prefix>)
+            # order matters, first match wins!
+            (r"^/usr/share/package-licenses/.{1,}/.{1,}", "license"),
+            (r"^/usr/share/man/man2", "dev"),
+            (r"^/usr/share/man/man3", "dev"),
+            (r"^/usr/share/man/", "man"),
+            (r"^/usr/share/pkgconfig/32.*\.pc$", "dev32"),
+            (r"^/usr/share/pkgconfig/", "dev"),
+            (r"^/usr/share/info/", "info"),
+            (r"^/usr/share/abi/", "abi"),
+            (r"^/usr/share/qt5/examples/", "examples"),
+            (r"^/usr/share/qt6/examples/", "examples"),
+            (r"^/usr/share/omf", "main", "/usr/share/omf/*"),
+            (r"^/usr/share/installed-tests/", "tests"),
+            (r"^/usr/libexec/installed-tests/", "tests"),
+            (r"^/usr/lib/rustlib/x86_64-unknown-linux-gnu/lib/[a-zA-Z0-9._+-]+\.rlib", "lib", "/usr/lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib"),
+            (r"^/usr/lib/rustlib/x86_64-unknown-linux-gnu/analysis/[a-zA-Z0-9._+-]+\.json", "lib", "/usr/lib/rustlib/x86_64-unknown-linux-gnu/analysis/*.json"),
+            (r"^/usr/share/clear/optimized-elf/bin", "bin", "/usr/share/clear/optimized-elf/bin*"),
+            (r"^/usr/share/clear/optimized-elf/exec", "libexec", "/usr/share/clear/optimized-elf/exec*"),
+            (r"^/usr/share/clear/optimized-elf/lib", "lib", "/usr/share/clear/optimized-elf/lib*"),
+            (r"^/usr/share/clear/optimized-elf/other", "lib", "/usr/share/clear/optimized-elf/other*"),
+            (r"^/usr/share/clear/optimized-elf/test", "tests", "/usr/share/clear/optimized-elf/test*"),
+            (r"^/usr/share/clear/optimized-elf/", "lib"),
+            (r"^/usr/share/clear/filemap/", "filemap"),
+            (r"^/usr/lib64/openmpi/bin/", "openmpi"),
+            (r"^/usr/lib64/openmpi/share", "openmpi"),
+            (r"^/usr/lib64/openmpi/include/", "dev"),
+            (r"^/usr/lib64/openmpi/lib/[a-zA-Z0-9._+-]*\.so$", so_dest_ompi),
+            (r"^/usr/lib64/openmpi/lib/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/openmpi/lib/[a-zA-Z0-9._+-]*\.so\.", "openmpi"),
+            (r"^/usr/lib64/openmpi/lib/python3.*/", "openmpi"),
+            (r"^/usr/lib64/openmpi/lib/", "dev"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.so\.", "plugins"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.so\.", "lib"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.so\.", "lib32"),
+            (r"^/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so", "lib"),
+            (r"^/usr/lib64/libkdeinit5", "lib"),
+            (r"^/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so", "lib32"),
+            (r"^/usr/lib64/haswell/[a-zA-Z0-9._+-]*\.so\.", "lib"),
+            (r"^/usr/lib64/gobject-introspection/", "lib"),
+            (r"^/usr/libexec/", "libexec"),
+            (r"^/usr/bin/", "bin"),
+            (r"^/usr/sbin/", "bin"),
+            (r"^/sbin/", "bin"),
+            (r"^/bin/", "bin"),
+            (r"^/usr/lib/python3.*/", "python3", "/usr/lib/python3*/*"),
+            (r"^/usr/share/gir-[0-9\.]+/[a-zA-Z0-9._+-]*\.gir", "data", "/usr/share/gir-1.0/*.gir"),
+            (r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"),
+            (r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"),
+            (r"^/usr/share/cmake-3.7/", "data", "/usr/share/cmake-3.7/*"),
+            (r"^/usr/share/cmake-3.8/", "data", "/usr/share/cmake-3.8/*"),
+            (r"^/usr/share/cmake-3.6/", "data", "/usr/share/cmake-3.6/*"),
+            (r"^/usr/share/girepository-1\.0/.*\.typelib\$", "data", "/usr/share/girepository-1.0/*.typelib"),
+            (r"^/usr/include/", "dev"),
+            (r"^/usr/lib64/girepository-1.0/", "data"),
+            (r"^/usr/share/cmake/", "dev"),
+            (r"^/usr/lib/cmake/", "dev"),
+            (r"^/usr/lib64/cmake/", "dev"),
+            (r"^/usr/lib32/cmake/", "dev32"),
+            (r"^/usr/lib/qt5/mkspecs/", "dev"),
+            (r"^/usr/lib/qt6/mkspecs/", "dev"),
+            (r"^/usr/lib64/qt5/mkspecs/", "dev"),
+            (r"^/usr/lib32/qt5/mkspecs/", "dev32"),
+            (r"^/usr/lib64/qt6/mkspecs/", "dev"),
+            (r"^/usr/lib32/qt6/mkspecs/", "dev32"),
+            (r"^/usr/lib/qt5/", "lib"),
+            (r"^/usr/lib/qt6/", "lib"),
+            (r"^/usr/lib64/qt5/", "lib"),
+            (r"^/usr/lib32/qt5/", "lib32"),
+            (r"^/usr/lib64/qt6/", "lib"),
+            (r"^/usr/lib32/qt6/", "lib32"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/libkdeinit5_[a-zA-Z0-9._+-]*\.so$", "lib"),
+            (r"^/usr/lib32/libkdeinit5_[a-zA-Z0-9._+-]*\.so$", "lib32"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.so$", so_dest + '32'),
+            (r"^/usr/lib64/glibc-hwcaps/x86-64-v[0-9]+/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/haswell/avx512_1/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/haswell/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib64/haswell/avx512_1/[a-zA-Z0-9._+-]*\.so$", so_dest),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.a$", "staticdev32"),
+            (r"^/usr/lib/haswell/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/glibc-hwcaps/x86-64-v[0-9]+/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/haswell/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib64/haswell/avx512_1/[a-zA-Z0-9._+-]*\.a$", "staticdev"),
+            (r"^/usr/lib32/haswell/[a-zA-Z0-9._+-]*\.a$", "staticdev32"),
+            (r"^/usr/lib/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib64/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib32/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev32"),
+            (r"^/usr/lib64/glibc-hwcaps/x86-64-v[0-9]+/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib64/haswell/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib64/haswell/avx512_1/pkgconfig/[a-zA-Z0-9._+-]*\.pc$", "dev"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.la$", "dev"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.la$", "dev"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.la$", "dev32"),
+            (r"^/usr/lib/[a-zA-Z0-9._+-]*\.prl$", "dev"),
+            (r"^/usr/lib64/[a-zA-Z0-9._+-]*\.prl$", "dev"),
+            (r"^/usr/lib32/[a-zA-Z0-9._+-]*\.prl$", "dev32"),
+            (r"^/usr/share/aclocal/[a-zA-Z0-9._+-]*\.ac$", "dev", "/usr/share/aclocal/*.ac"),
+            (r"^/usr/share/aclocal/[a-zA-Z0-9._+-]*\.m4$", "dev", "/usr/share/aclocal/*.m4"),
+            (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9._+-]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"),
+            (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9._+-]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"),
+            (r"^/usr/share/doc/" + re.escape(pkg_name) + "/", "doc", "/usr/share/doc/" + re.escape(pkg_name) + "/*"),
+            (r"^/usr/share/doc/", "doc"),
+            (r"^/usr/share/gtk-doc/html", "doc"),
+            (r"^/usr/share/help", "doc"),
+            (r"^/usr/share/info/", "doc", "/usr/share/info/*"),
+            # now a set of catch-all rules
+            (r"^/lib/systemd/system/", "services"),
+            (r"^/lib/systemd/user/", "services"),
+            (r"^/usr/lib/systemd/system/", "services"),
+            (r"^/usr/lib/systemd/user/", "services"),
+            (r"^/usr/lib/udev/rules.d", "config"),
+            (r"^/usr/lib/modules-load.d", "config"),
+            (r"^/usr/lib/tmpfiles.d", "config"),
+            (r"^/usr/lib/sysusers.d", "config"),
+            (r"^/usr/lib/sysctl.d", "config"),
+            (r"^/usr/share/", "data"),
+            (r"^/usr/lib/perl5/", "perl", "/usr/lib/perl5/*"),
+            # finally move any dynamically loadable plugins (not
+            # perl/python/etc.. extensions) into lib package
+            (r"^/usr/lib/.*/[a-zA-Z0-9._+-]*\.so", "lib"),
+            (r"^/usr/lib64/.*/[a-zA-Z0-9._+-]*\.so", "lib"),
+            (r"^/usr/lib32/.*/[a-zA-Z0-9._+-]*\.so", "lib32"),
+            # locale data gets picked up via file_is_locale
+            (r"^/usr/share/locale/", "ignore")]
+
+        for pat_args in patterns:
+            if self.file_pat_match(filename, *pat_args):
+                return
+
+        if filename in self.excludes:
+            return
+
+        self.push_package_file(filename)
+
+    def remove_file(self, filename):
+        """Remove filename from local file list."""
+        hit = False
+
+        if filename in self.files:
+            self.files.remove(filename)
+            print("File no longer present: {}".format(filename))
+            hit = True
+        for pkg in self.packages:
+            if filename in self.packages[pkg]:
+                self.packages[pkg].remove(filename)
+                print("File no longer present in {}: {}".format(pkg, filename))
+                hit = True
+        if hit:
+            self.files_blacklist.add(filename)
+            self.package.must_restart += 1
+
+    def load_specfile(self, specfile):
+        """Load a specfile instance with relevant information to be written to the spec file."""
+        specfile.packages = self.packages
+        specfile.excludes = self.excludes
+        specfile.locales = self.locales
+        specfile.file_maps = self.file_maps
+        specfile.setuid = self.setuid

autospec/git.py

@@ -19,51 +19,150 @@
 # Commit to git
 #
 
+import glob
+import os
 import subprocess
+import sys
+import tempfile
 
-import build
+from util import call, open_auto, write_out
-import buildpattern
-import tarball
-import test
-import docs
-from util import call
-import config
 
 
-def commit_to_git(path):
+def get_autospec_info():
-
+    """Get the latest tag from autospec."""
-    if not config.git_uri:
+    path = os.path.dirname(sys.path[0])
-        return
+    git_out = tempfile.mkstemp()[1]
-
-    call("git init", stdout=subprocess.DEVNULL, cwd=path)
-
     try:
-        call("git config --get remote.origin.url", cwd=path)
+        call("git tag -l --sort=-v:refname", git_out, cwd=path)
-    except subprocess.CalledProcessError:
+        with open_auto(git_out) as gfile:
-        upstream_uri = config.git_uri % {'NAME': tarball.name}
+            tag = gfile.readlines()[0].strip()
-        call("git remote add origin %s" % upstream_uri, cwd=path)
+    except Exception:
+        tag = ""
+    os.unlink(git_out)
+    try:
+        call('git log -1 --pretty=format:"%h"', git_out, cwd=path)
+        with open_auto(git_out) as gfile:
+            commit = gfile.readlines()[0].strip()
+    except Exception:
+        commit = ""
+    os.unlink(git_out)
+    return tag, commit
+
+
+def commit_to_git(config, name, success):
+    """Update package's git tree for autospec managed changes."""
+    path = config.download_path
+    call("git init -b main", stdout=subprocess.DEVNULL, cwd=path)
+
+    # This config is used for setting the remote URI, so it is optional.
+    if config.git_uri:
+        try:
+            call("git config --get remote.origin.url", cwd=path)
+        except subprocess.CalledProcessError:
+            upstream_uri = config.git_uri % {'NAME': name}
+            call("git remote add origin %s" % upstream_uri, cwd=path)
 
     for config_file in config.config_files:
-        call("git add %s" % config_file, cwd=path)
+        call("git add %s" % config_file, cwd=path, check=False)
-    for unit in buildpattern.sources["unit"]:
+    for unit in config.sources["unit"]:
         call("git add %s" % unit, cwd=path)
     call("git add Makefile", cwd=path)
     call("git add upstream", cwd=path)
-    call("git add *.spec", cwd=path)
+    call("bash -c 'shopt -s failglob; git add *.spec'", cwd=path)
-    if test.unit_pass_written:
+    call("git add %s.tmpfiles" % name, check=False, stderr=subprocess.DEVNULL, cwd=path)
-        call("git add unit_tests_must_pass", cwd=path)
+    call("git add %s.sysusers" % name, check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add %s.tmpfiles" % tarball.name, check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add prep_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add pypi.json", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add funroll-loops", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add build_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add make_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add install_prepend", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add series", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    # Add/remove version specific patch lists
+    for filename in glob.glob('series.*'):
+        base, version = filename.split('.', 1)
+        if version in config.versions:
+            call("git add {}".format(filename), check=False, stderr=subprocess.DEVNULL, cwd=path)
+        else:
+            call("git rm {}".format(filename), check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.asc'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.sig'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.sha256'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.sign'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *.pkey'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure32", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure64", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure_avx2", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure_avx512", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add make_check_command", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add keepstatic", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add *.patch'", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    call("git add *.patch", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add *.nopatch'", check=False, stderr=subprocess.DEVNULL, cwd=path)
-    for item in docs.transforms.values():
+    for item in config.transforms.values():
         call("git add {}".format(item), check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add release", cwd=path)
+    call("git add symbols", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add symbols32", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add used_libs", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add used_libs32", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add testresults", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add profile_payload", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add options.conf", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add configure_misses", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add etc_files", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add whatrequires", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add description", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add attrs", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add archive.diff", check=False, stderr=subprocess.DEVNULL, cwd=path)
 
-    if build.success == 0:
+    # remove deprecated config files
+    call("git rm make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm prep_append", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm use_clang", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm use_lto", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm use_avx2", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm fast-math", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm broken_c++", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm skip_test_suite", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm optimize_size", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm asneeded", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm broken_parallel_build", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm pgo", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm unit_tests_must_pass", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm funroll-loops", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm keepstatic", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm allow_test_failures", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm no_autostart", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm insecure_build", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git rm conservative_flags", check=False, stderr=subprocess.DEVNULL, cwd=path)
+
+    # add a gitignore
+    ignorelist = [
+        ".*~",
+        "*~",
+        "*.info",
+        "*.mod",
+        "*.swp",
+        ".repo-index",
+        "*.log",
+        "build.log.round*",
+        "*.tar.*",
+        "*.tgz",
+        "!*.tar.*.*",
+        "*.zip",
+        "*.jar",
+        "*.pom",
+        "*.xml",
+        "commitmsg",
+        "results/",
+        "rpms/",
+        "for-review.txt",
+        ""
+    ]
+    write_out(os.path.join(path, '.gitignore'), '\n'.join(ignorelist))
+    call("git add .gitignore", check=False, stderr=subprocess.DEVNULL, cwd=path)
+
+    if success == 0:
         return
 
     call("git commit -a -F commitmsg ", cwd=path)

autospec/ignored_commands

@@ -0,0 +1,599 @@
+# This file contains ignored commands from configure output
+# in the form: <configure line>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+# For patterns that start with '#', escape the '#' as '\#'.
+-Wmissing-braces
+-Wparentheses-equality
+-Wtypedef-redefinition
+-Wunknown-attributes
+-Wunknown-warning-option
+-lc should be explicitly linked in
+-ldld
+-lmkl
+-lposix1e
+-lsec
+-lselinux
+-lsmack
+-lsocket
+-lxpg4
+// is distinct from /
+/dev/ptc
+/etc/SuSE-release
+/etc/debian_version
+/etc/fedora-release
+/etc/gentoo-release
+/etc/mandriva-release
+/etc/redhat-release
+<limits.h> defines MIN and MAX
+<pthread.h> pollutes the namespace
+AIX ACL
+ANSI C header files
+ARM NEON in current arch/CFLAGS
+Avahi
+AvailabilityMacros.h
+BeOS
+C compiler accepts -Wcomma
+C compiler accepts -Werror=unknown-warning-option
+C compiler accepts -Winitializer-overrides
+C compiler handles -Werror -Wunknown-warning-option
+C symbols are prefixed with underscore at the linker level
+CC
+CDPARANOIA
+CELT051
+CFLocaleCopyCurrent
+CFPreferencesCopyAppValue
+CYGWIN or MSYS environment
+Darwin (Mac OS X)
+Darwin (Mac OS X) platform
+ELOGIND
+GTK_QUARTZ
+HAVE_BIG_ENDIAN
+HAVE_INCOHERENT_MMAP
+HDF5 library has enforced version 1.6 API
+HP-UX
+KF5
+LIBPSL
+LIBTEAMDCTL
+LUA_PREFIX
+Minix Amsterdam compiler
+MobileCoreServices/MobileCoreServices.h
+OS.h
+QNX
+Qt5WebKitWidgets
+SELINUX
+Solaris 11.4 locale system
+Solaris ACL
+Ubuntu
+Unknown pattern match:
+WIN32
+Win32
+Win32 platform in general
+XATTR_NAME_SMACKEXEC in linux/xattr.h
+XATTR_NOFOLLOW
+XML catalog (/etc/xml/catalog)
+XSI (rather than GNU) prototype for strerror_r
+_ prefix in compiled symbols
+_FILE_OFFSET_BITS value needed for large files
+_FILE_OFFSET_BITS value needed large files
+_LARGEFILE_SOURCE value needed for large files
+_LARGEFILE_SOURCE value needed large files
+_NSGetEnviron
+__ApplePrivate_gss_acquire_cred_ex_f
+__INTEL_COMPILER
+__SUNPRO_C
+__argv
+__clang__
+__freadahead
+__freadptr
+__freadptrinc
+__fseterr
+__i386__
+__mktime_internal
+__popcnt
+__popcnt64
+__res_init
+__secure_getenv
+__strtoll
+__strtoull
+__sys_siglist
+__uint128_t
+__xpg4
+_byteswap_ulong
+_byteswap_ushort
+_doprnt
+_fseeki64
+_getpid
+_putenv_s
+_set_invalid_parameter_handler
+_snprintf
+_strdup
+_xpg4_setrunelocale in -lxpg4
+a
+a statically linked program can dlopen itself
+a turkish Unicode locale
+abs
+acl_get_file in -lposix1e
+acos
+acosh
+any prototype needed for pread
+any prototype needed for pwrite
+ar
+arithmetic hrtime_t
+as
+asin
+asinh
+atan
+atanh
+atomic_add_32 compiler builtin
+bcopy
+binary relocation should be enabled
+broken poll
+bstring.h
+build strict
+byte ordering is bigendian
+c++
+cbrt
+cc
+ceil
+chsize
+clang option to accept ISO C89
+clock_gettime
+closedir returns void
+closefrom
+compiler driver understands Ada
+compilers -Wno-format-truncation
+compilers -Wno-tautological-compare
+compilers -Wno-unused-result
+compilers -Wsign-conversion
+cos
+cosh
+crt_externs.h
+crtdefs.h
+crypt
+crypt_r
+cygwin
+declaration of PTHREAD_MUTEX_ROBUST
+default print group
+default print user
+dependency style of gcc
+desired default level of POSIX conformance
+directio
+dirfd is a macro
+dl.h
+dlclose
+dld_link in -ldld
+dlerror
+dlltool
+dlopen
+dlopen()
+dlsym
+dns_sd.h
+dustat.h
+eatmydata
+epydoc
+erf
+erfc
+errno_t
+exp
+explicit OBJCFLAGS
+external symbol _system_configuration
+extra flags to get ANSI library prototypes
+fabs
+fake locale system (OpenBSD)
+fallocate
+fdatasync
+fdatasync... (cach
+fdwalk
+fflush works on input streams
+filefrag
+floor
+fop
+for gcc option to accept ISO C89
+for minix/config.h
+for mt
+for sysroot
+for x86_64-generic-linux-gnu-dlltool
+for x86_64-generic-linux-gnu-mt
+for x86_64-generic-linux-gnu-objdump
+for x86_64-generic-linux-gnu-strip
+fpurge
+fpurge without a macro
+frame.h
+g++ option to enable C++11 features
+gai_strerrorA
+gcc
+gcc accepts -g
+gcc is Clang
+gcc needs -traditional
+gcc option to accept ANSI C
+gcc option to accept ISO C89
+gcc option to accept ISO C99
+gcc option to enable C11 features
+gcc options needed to detect all undeclared functions
+gcc understands --disable-instructions
+gcc understands --omg-optimized
+gcc understands -Wold-style-definition
+gcc understands -Wstrict-prototypes
+gccgo
+gccs -Wall
+gccs -mpower8-vector
+gdiff
+getaddrinfo
+getattrat
+getcwd
+getcwd aborts when 4k < cwd_length < 16k
+getdents
+getexecname
+getgrent_nomembers
+getgrgid_nomembers
+getgrnam_nomembers
+gethostbyaddr
+gethostbyname
+gethostbyname... (cach
+gethrtime
+getifaddrs
+getifaddrs() and AF_LINK
+getlocalename_l
+getmntent
+getmntinfo
+getnameinfo
+getpeereid
+getpeerucred
+getppriv
+getprogname
+getpseudotty
+gets without a macro
+getservbyname
+getservbyname... (cach
+getspent
+getsysinfo
+gettimeofday clobbers localtime buffer
+gm4
+gnatbind
+gnatmake
+gnudiff
+gnum4
+gtar
+history.h
+how to do getaddrinfo, freeaddrinfo and getnameinfo... checking for getaddrinfo
+how to run the C preprocessor
+hstrerror
+hurd.h
+hypot
+icc
+iconv_open
+ieeefp.h
+if -lm is required for maths functions
+if : is a manifest tool
+if GCC is actually Clang
+if X11 header
+if X11 header files implicitly declare return values
+if building for some Win32 platform
+if building the Windows installer
+if character set is EBCDIC
+if compiler needs -Werror to reject unknown flags
+if compiler supports -R
+if compiling for Mac OS X
+if compiling for Win32
+if compiling with clang
+if debug is enabled
+if g++ static flag -static works
+if gcc static flag -static works
+if gcc supports -fno-rtti -fno-exceptions
+if gccs -Werror=unknown-warning-option
+if gccs -Werror=unused-command-line-argument
+if gccs -fno-rtti -fno-exceptions
+if internal cryptsetup PBKDF2 is compiled-in
+if more special flags are required for pthreads
+if zts is enabled
+included gettext is requested
+inet_ntop
+inet_ntop... (cach
+inet_pton
+install_name_tool
+integer division by zero raises SIGFPE
+io.h
+ioctl with POSIX signature
+is_selinux_enabled in -lselinux
+isapipe
+j0
+j1
+kqueue
+launch.h
+launch_activate_socket
+lchmod
+ld
+libc.h
+libiconv_open
+libnet.h version 1.1.x
+library containing opendir
+library containing strerror
+library setproctitle
+libsigsegv
+libunwind-ia64.h
+libutil.h
+link(2) dereferences a symlink
+lipo
+listmntent
+listmntent of Cray/Unicos-9
+localtime caches TZ
+localtime loops forever near extrema
+log
+log10
+login_tty
+long double and double are the same
+lstat accepts an empty string
+mach-o/dyld.h
+mach/host_info.h
+mach/mach_time.h
+machine/hal_sysinfo.h
+main in -lelf
+major_t
+matchpathcon_init_prefix
+mbslen
+mbswidth in <wchar.h>
+member __ss_family in struct sockaddr_storage
+member sa_len in struct sockaddr
+member sin6_len in struct sockaddr_in6
+member sin_len in struct sockaddr_in
+memcpy
+memory.h
+memset_s
+microuptime
+minix/config.h
+minix/config.h presence
+minix/config.h usability
+minor_t
+mntctl function and struct vmount
+mode_to_security_class
+more special flags are required for pthreads
+more warnings
+mt
+multithread API to use
+mxml.h
+nanosleep
+nanotime
+nanouptime
+native Win32
+native macOS
+net/bpf.h
+net/if_dl.h
+net/pfilt.h
+net/pfvar.h
+net/raw.h
+nfs/vfs.h
+nm
+objc
+objcc
+objcopy
+objdump
+obstacks that work with any size object
+offset_t
+opendir
+openpty
+org
+plugin documentation
+port_create
+posix_spawn
+precise C stack overflow detection
+printfs the 'n' directive
+priv.h
+process.h
+pstat_getdynamic
+pstat_getstatic
+pthread_create
+pthread_create using -lpthreads
+pthread_mutex_consistent
+pthread_mutexattr_setrobust
+pthread_np.h
+pthread_rwlock_rdlock prefers a writer to a reader
+pthread_setname_np(const char*)
+pthread_sigmask
+pthread_sigmask is only a macro
+pthreads work with -Kthread
+pthreads work with -kthread
+pthreads work without any flags
+putmsg in -lstr
+py.test-2.7
+pyinotify
+python2.6
+ranlib
+readelf
+removefile
+res_search
+rint
+rresvport_af
+sched_getaffinity_np
+selinux/context.h
+selinux/flask.h
+selinux/selinux.h
+setfilecon
+setproctitle
+setproctitle_init
+setsid
+setsockopt
+shared libraries
+shl_findsym
+shl_load
+shl_load in -ldld
+shl_unload
+sig2str
+sighandler_t
+sighandlers must be reinstalled
+sigsetjmp
+sigvec
+sinh
+size_t
+smack_new_label_from_self in -lsmack
+snprintf
+socket
+socket in -lsocket
+socketpair
+some Darwin platform
+some Win32 platform
+special C compiler options needed for large files
+special C compiler options needed large files
+sqrt
+st_dm_mode in struct stat
+st_gen member in stat structure
+stack_t has an ss_base field
+stat accepts an empty string
+stat() ignores a trailing slash
+statfs that truncates block counts
+static libraries
+stdint.h
+stdint.h predates C++11
+strerror
+strerror_r with POSIX signature
+strerror_s
+stricmp
+strip
+strlcat
+strlcpy
+strmode
+strncpy_s
+strnicmp
+strnstr
+struct dirent.d_namlen
+struct fsstat.f_fstypename
+struct sockaddr.sa_len
+struct sockaddr_in.sin_len
+struct sockaddr_in6.sin6_len
+struct sockaddr_storage.__ss_family
+struct stat
+struct stat.st_atimensec
+struct stat.st_author
+struct stat.st_birthtim.tv_nsec
+struct stat.st_birthtime
+struct stat.st_birthtimensec
+struct stat.st_birthtimespec.tv_nsec
+struct stat.st_ctimensec
+struct stat.st_flags
+struct stat.st_mtimensec
+struct stat.st_mtimespec.tv_nsec
+struct statfs.f_fstypename
+struct statfs.f_namemax
+struct statvfs.f_basetype
+struct statvfs.f_fstypename
+struct statvfs.f_type
+struct tm.__tm_gmtoff
+struct utmp.ut_exit.ut_exit
+struct utmp.ut_exit.ut_termination
+struct utmpx.ut_exit.ut_exit
+struct utmpx.ut_exit.ut_termination
+suncc
+symbols are prefixed
+sys/access.h
+sys/dirent.h
+sys/dlpi.h
+sys/endian.h
+sys/event.h
+sys/filio.h
+sys/fs/s5param.h
+sys/fs_types.h
+sys/ioccom.h
+sys/loadavg.h
+sys/mkdev.h
+sys/mntent.h
+sys/mnttab.h
+sys/pstat.h
+sys/ptem.h
+sys/sockio.h
+sys/stream.h
+sys/sysmp.h
+sys/systemcfg.h
+sys/systeminfo.h
+sys/table.h
+sys/thr.h
+sys/types.h defines makedev
+sys/ucred.h
+sys/utime.h
+sysctl
+sysctlbyname
+sysmouse
+sysmp
+sysroot
+system header files limit the line length
+table
+tan
+tanh
+termios.h defines TIOCGWINSZ
+termios.h needs _XOPEN_SOURCE
+tgetflag
+the BeOS
+the C locale is free of encoding errors
+the compiler is actually Clang
+the linker accepts -Wl,-fatal_warnings
+the pthreads library -llthread
+the pthreads library -lpthreads
+the systems xlc include
+thr_self
+timezone_t
+to add custom build version
+to enable debugging
+to enable maintainer-specific portions of Makefiles
+to enable pch feature
+to sign windows build
+to treat the installation as read-only
+to turn on debugging
+to turn on profiling
+to turn warnings to errors
+to use C++
+to use statvfs64
+toupper
+tputs() uses outfuntype
+tzset clobbers localtime buffer
+uint_t
+unistd.h
+use of TIOCGWINSZ requires termios.h
+use of struct winsize requires sys/ptem.h
+util.h
+util/debug.h
+util/msg18n.h
+va_lists can be copied by value
+varargs.h
+variable rl_event_hook
+vasnprintf
+vfork.h
+w32api/winbase.h
+w32api/wtypes.h
+w3m
+we are cross compiling
+we are using the GNU C compiler
+we are using the GNU Objective C compiler
+we are using the GNU Objective C++ compiler
+we build a relocatable package
+we need to force -D_FILE_OFFSET_BITS=64
+we want to fetch tarballs
+wget
+whether -Wmissing-braces works as expected
+whether -Wparentheses-equality works as expected
+whether -Wtypedef-redefinition works as expected
+whether -Wunknown-attributes works as expected
+whether -Wunknown-warning-option works as expected
+whether -lc should be explicitly linked in
+whether to build static libraries
+whether we are cross compiling
+which package format to use
+wincrypt.h
+windmc
+windows.h
+windres
+winsock2.h
+wint_t is too small
+with code coverage
+with code coverage support
+with gcov testing
+working alloca.h
+working mmap
+working nanosleep
+ws2tcpip.h
+x86_64-generic-linux-gnu-dlltool
+x86_64-generic-linux-gnu-mt
+x86_64-generic-linux-gnu-objdump
+x86_64-generic-linux-gnu-strip
+xlocale.h
+xmkmf
+y0
+y1

autospec/keyid_blocklist

@@ -0,0 +1 @@
+59FCF207FEA7F445

autospec/lang.py

@@ -1,52 +0,0 @@
-#!/bin/true
-#
-# lang.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Parse config files
-#
-
-import files
-
-
-locales = []
-
-
-def add_lang(lang):
-    global locales
-    if lang in locales:
-        return
-    locales.append(lang)
-    print("  New locale:", lang)
-
-    if "locales" in files.packages:
-        return
-    files.packages["locales"] = []
-
-
-def write_find_lang(file):
-    for lang in locales:
-        file.write("%find_lang " + lang + "\n")
-
-
-def write_lang_files(file):
-    global locales
-    if len(locales) == 0:
-        return
-    file.write("\n%files locales ")
-    for lang in locales:
-        file.write("-f " + lang + ".lang ")
-    file.write("\n%defattr(-,root,root,-)\n\n")

autospec/license.py

@@ -22,256 +22,178 @@
 #
 
 import os
-import tarball
+import re
-import urllib
+import shlex
-import config
+import sys
+import urllib.parse
+
+import chardet
+import download
+from util import get_contents, get_sha1sum, print_fatal, print_warning
 
 default_license = "TO BE DETERMINED"
 
-LICENSE_HASH_FILE = "common/licenses"
-
 licenses = []
-
+license_files = []
-license_translations = {
+hashes = dict()
-    "GPL-2.0": "GPL-2.0",
-    "GPLv2": "GPL-2.0",
-    "GPLV2": "GPL-2.0",
-    "GPLV3": "GPL-3.0",
-    "GPL-2": "GPL-2.0",
-    "GPL-2+": "GPL-2.0+",
-    "GPL-2.0+": "GPL-2.0+",
-    "GPLv2+": "GPL-2.0+",
-    "GPL(>=2)": "GPL-2.0+",
-    "GPL(>=-2)": "GPL-2.0+",
-    "LGPL(>=2)": "LGPL-2.0+",
-    "LGPL(>=-2)": "LGPL-2.0+",
-    "LGPLv2": "LGPL-2.0",
-    "LGPLv2.1": "LGPL-2.1",
-    "LGPLv2+": "LGPL-2.1+",
-    "LGPL-2.0+": "LGPL-2.0+",
-    "LGPL-2.1": "LGPL-2.1",
-    "LGPLv2.1": "LGPL-2.1",
-    "LGPL-2.1+": "LGPL-2.1+",
-    "LGPLv2.1+": "LGPL-2.1+",
-    "LGPLv3+": "LGPL-3.0+",
-    "LGPLv3": "LGPL-3.0",
-    "GPL-3.0": "GPL-3.0",
-    "GPLv3": "GPL-3.0",
-    "GPL-3": "GPL-3.0",
-    "LGPL-3": "LGPL-3.0",
-    "zlib": "Zlib",
-    "zlib/libpng": "zlib-acknowledgement",
-    "Boost": "BSL-1.0",
-    "GPL-3.0+": "GPL-3.0+",
-    "GPLv3+": "GPL-3.0+",
-    "GPL3": "GPL-3.0",
-    "GPL(>=3)": "GPL-3.0+",
-    "http://opensource.org/licenses/MIT": "MIT",
-    "mit": "MIT",
-    "http://www.apache.org/licenses/LICENSE-2.0": "Apache-2.0",
-    "Apache License, Version 2.0": "Apache-2.0",
-    "Apache License 2.0": "Apache-2.0",
-    "APL2.0": "Apache-2.0",
-    "APL2": "Apache-2.0",
-    "ASL 2.0": "Apache-2.0",
-    "ASL-2.0": "Apache-2.0",
-    "APL-2.0": "Apache-2.0",
-    "ASL-2": "Apache-2.0",
-    "Apache2.0": "Apache-2.0",
-    "Apache 2.0": "Apache-2.0",
-    "Apache": "Apache-2.0",
-    "apache": "Apache-2.0",
-    "Apache-2": "Apache-2.0",
-    "artistic_2": "Artistic-2.0",
-    "MPLv1.1": "MPL-1.1",
-    "ZPL 2.1": "ZPL-2.1",
-    "ZPL": "ZPL-2.0",
-    "http://creativecommons.org/licenses/BSD/": "BSD-2-Clause",
-    "BSD_3_clause": "BSD-3-Clause",
-    "perl": "Artistic-1.0-Perl",
-    "PSF": "Python-2.0",
-    "Python": "Python-2.0",
-    "BSD_2_clause": "BSD-2-Clause",
-    "Expat": "MIT",
-    "VIM": "Vim"
-}
-
-license_blacklist = {
-    "and": True,
-    "BSD": True,
-    "3BSD": True,
-    "LGPL": True,
-    "GPL": True,
-    "ASL": True,
-    "2.0": True,
-    "advertising": True,
-    "LGPL+BSD": True,
-    "UN": True,
-    "GNU": True,
-    "new": True,
-    "none": True,
-    "License": True,
-    "License": True,
-    "license": True,
-    "Standard": True,
-    "PIL": True,
-    "Software": True,
-    "|": True,
-    "+": True,
-    "UNKNOWN": True,
-    "unknown": True,
-    "BSD-like": True,
-    "or": True,
-    "Modified": True,
-    "3-clause": True,
-    "Unlimited": True,
-    "BSD_3_clause": True,
-    "BSD(3": True,
-    "clause)": True,
-    "GFDL": True,
-    "MPL": True,
-    "Muddy-MIT": True,
-    "LGPL/MIT": True,
-    "License,": True,
-    "LICENCE": True,
-    "-": True,
-    "See": True,
-    "details": True,
-    "for": True,
-    "Version-2.0": True,
-    "exceptions": True,
-    "http://nmap.org/man/man-legal.html": True,
-    "with": True,
-    "style": True,
-    "Foundation": True,
-    "~": True,
-    "open_source": True,
-    "BSDish": True,
-    "EPL": True,
-    "Artistic": True,
-    "(specified": True,
-    "using": True,
-    "classifiers)": True,
-    "APL-2.0": True,
-    "dual": True,
-    "GPL/BSD": True
-}
 
 
-def add_license(lic):
+def process_licenses(lics, translations, blacklist):
+    """Handle licenses string from the license server.
+
+    The license server response may contain multiple space-separated licenses.
+    Add each license individually.
+    """
+    for lic in lics.split():
+        add_license(lic, translations, blacklist)
+
+
+def add_license(lic, translations, blacklist):
+    """Add licenses from the server.
+
+    Add license from license string lic after checking for duplication or
+    presence in the blacklist. Returns False if no license were added, True
+    otherwise.
+    """
     global licenses
-    lic = lic.strip()
+    global license_files
+    lic = lic.strip().strip(',')
+    result = False
 
-    # Bail if the license isn't one we recognize, is a blacklisted word or has already been added
+    # Translate the license if a translation exists
-    if lic in licenses:
+    real_lic_str = translations.get(lic, lic)
-        return False
+    real_lics = real_lic_str.split()
-    if lic in license_blacklist:
+    for real_lic in real_lics:
-        return False
+        if real_lic in blacklist:
-    if lic not in license_translations:
+            continue
-        licenses.append(lic)
+        elif real_lic in licenses:
-        return True
+            result = True
-    if license_translations[lic] in licenses:
+        else:
-        return False
+            result = True
+            licenses.append(real_lic)
 
-    licenses.append(license_translations[lic])
+    return result
-    return True
 
 
-def license_from_copying_content(copying):
+def decode_license(license):
-    with open(copying, 'r', encoding="latin-1") as content_file:
+    """Try and decode the license string."""
-        content = content_file.read()
+    def try_with_charset(license, charset):
-    if content.find("Version 2, June 1991") >= 0:
+        if not charset:
-        add_license("GPL-2.0")
+            return
-    if content.find("Version 3, 29 June 2007") >= 0:
+
-        add_license("GPL-3.0")
+        try:
-    if content.find("Version 2.1, February 1999") >= 0:
+            return license.decode(charset)
-        add_license("LGPL-2.1")
+        except UnicodeDecodeError:
-    if content.find("Source Code is licensed under MIT license") >= 0:
+            if charset in ('ISO-8859-1', 'ISO-8859-15'):
-        add_license("MIT")
+                if b'\xff' in license:
-    if content.find("Version 2.0, January 2004") >= 0:
+                    return try_with_charset(license, 'ISO-8859-13')
-        add_license("Apache-2.0")
+                if b'\xd2' in license and b'\xd3' in license:
+                    return try_with_charset(license, 'mac_roman')
+
+    return try_with_charset(license, chardet.detect(license)['encoding'])
 
 
-def license_from_copying_hash(copying):
+def license_from_copying_hash(copying, srcdir, config, name):
-    """Add licenses based on the hash of the copying file"""
+    """Add licenses based on the hash of the copying file."""
-    licenses_list = []
+    try:
-    hash_sum = tarball.get_sha1sum(copying)
+        data = get_contents(copying)
+    except FileNotFoundError:
+        # LICENSE file is a bad symlink (qemu-4.2.0!)
+        return
 
-    licenses_dict = dict()
+    if data.startswith(b'#!'):
+        # Not a license if this is a script
+        return
+
+    data = decode_license(data)
+    if not data:
+        return
+
+    hash_sum = get_sha1sum(copying)
 
     if config.license_fetch:
-        with open(copying, "r", encoding="latin-1") as myfile:
+        values = {'hash': hash_sum, 'text': data, 'package': name}
-            data = myfile.read()
+        data = urllib.parse.urlencode(values)
-            url = config.license_fetch
+        data = data.encode('utf-8')
-            values = {'hash': hash_sum, 'text': data, 'package': tarball.name}
-            data = urllib.parse.urlencode(values)
-            data = data.encode('utf-8')
-            req = urllib.request.Request(url, data)
-            response = urllib.request.urlopen(req)
-            the_page = response.read().decode('utf-8')
-            if len(the_page.strip()) > 0:
-                print("License     : ", the_page.strip(), " (server) (", hash_sum, ")")
-                add_license(the_page.strip())
-                return
-    else:
-        if os.path.exists(LICENSE_HASH_FILE):
-            with open(LICENSE_HASH_FILE, "r") as file:
-                licenses_list = file.readlines()
-        licenses_dict = dict(license.split(" | ") for license in licenses_list)
 
-    if hash_sum in licenses_dict:
+        buffer = download.do_curl(config.license_fetch, post=data, is_fatal=True)
-        add_license(licenses_dict[hash_sum])
+        response = buffer.getvalue()
+        page = response.decode('utf-8').strip()
+        if page:
+            print("License     : ", page, " (server) (", hash_sum, ")")
+            process_licenses(page, config.license_translations, config.license_blacklist)
+
+            if page != "none":
+                # Strip the build source directory off the front
+                lic_path = copying[len(srcdir):]
+                # Strip any leading slashes
+                while lic_path.startswith('/'):
+                    lic_path = lic_path[1:]
+                lic_path = shlex.quote(lic_path)
+                license_files.append(lic_path)
+                hashes[lic_path] = hash_sum
+
+            return
+
+    if hash_sum in config.license_hashes:
+        add_license(config.license_hashes[hash_sum],
+                    config.license_translations,
+                    config.license_blacklist)
     else:
         if not config.license_show:
             return
-        print("Unknown license {0} with hash {1}".format(copying, hash_sum))
+        print_warning("Unknown license {0} with hash {1}".format(copying, hash_sum))
-        hashUrl = config.license_show % {'HASH': hash_sum}
+        hash_url = config.license_show % {'HASH': hash_sum}
-        print("Visit {0} to enter".format(hashUrl))
+        print_warning("Visit {0} to enter".format(hash_url))
 
 
-def license_from_doc(doc):
+def scan_for_licenses(srcdir, config, pkg_name):
-    """ Scan for documentation license in the given file """
+    """Scan the project directory for things we can use to guess a description and summary."""
-    with open(doc, 'r', encoding="latin-1") as content_file:
+    targets = ["copyright",
-        content = content_file.read()
+               "copyright.txt",
-    if content.find("GNU Free Documentation License, Version 1.3") >= 0:
+               "apache-2.0",
-        add_license("GFDL-1.3")
+               "artistic.txt",
-    elif content.find("GNU Free Documentation License, Version 1.2") >= 0:
+               "libcurllicense",
-        add_license("GFDL-1.2")
+               "gpl.txt",
-    elif content.find("GNU Free Documentation License, Version 1.1") >= 0:
+               "gpl2.txt",
-        add_license("GFDL-1.1")
+               "gplv2.txt",
-    elif content.find("(The MIT License)") >= 0:
+               "notice",
-        add_license("MIT")
+               "copyrights",
-
+               "about_bsd.txt"]
-
+    # look for files that start with copying or licen[cs]e (but are
-#
+    # not likely scripts) or end with licen[cs]e
-# Scan the project directory for things we can use to guess a description
+    target_pat = re.compile(r"^((copying)|(licen[cs]e)|(e[dp]l-v\d+))|(licen[cs]e)(\.(txt|xml))?$")
-# and summary
+    for dirpath, dirnames, files in os.walk(srcdir):
-#
-
-def scan_for_licenses(package, dir):
-    for dirpath, dirnames, files in os.walk(dir):
         for name in files:
-            if (name.lower() in ("copyright", "apache-2.0", "libcurllicense", "gpl.txt", "gplv2.txt", "notice", "copyrights", "about_bsd.txt") or
+            if name.lower() in targets or target_pat.search(name.lower()):
-               name.lower().startswith("copying") or name.lower().startswith("license") or name.lower().endswith("license") or name.lower().startswith("licence") or name.lower().endswith("licence")):
+                license_from_copying_hash(os.path.join(dirpath, name),
-                license_from_copying_hash(os.path.join(dirpath, name))
+                                          srcdir, config, pkg_name)
-                license_from_copying_content(os.path.join(dirpath, name))
+            # Also search for license texts in project trees that are
-            else:
+            # REUSE-compliant, or are in process of adopting this standard (for
-                if "." not in name:
+            # example, KDE ecosystem packages). See https://reuse.software for
-                    continue
+            # details. At a basic level, this layout requires a toplevel
-                ext = name.split(".")[-1].lower()
+            # `LICENSES` directory that includes separate files (with .txt
-                if ext in ("man", "texi", "rdoc"):
+            # extension) for each license text that covers source code, data,
-                    license_from_doc(os.path.join(dirpath, name))
+            # etc elsewhere in the project tree. A variant layout is currently
+            # seen in the DPDK 20.11.3 tree, where the `LICENSES` directory is
+            # named `license` instead.
+            dirbase = os.path.basename(dirpath)
+            if re.search(r'^(LICENSES|licenses?)$', dirbase) and re.search(r'\.txt$', name):
+                license_from_copying_hash(os.path.join(dirpath, name),
+                                          srcdir, config, pkg_name)
 
-    if len(licenses) == 0:
+    if not licenses:
-        print("\n*ERROR: cannot find any license or " + tarball.name + ".license file!\n")
+        print_fatal(" Cannot find any license or a valid {}.license file!\n".format(pkg_name))
-        quit()
+        sys.exit(1)
-    else:
+
-        print("Licenses    : ", " ".join(sorted(licenses)))
+    print("Licenses    : ", " ".join(sorted(licenses)))
 
 
-def write_license(file):
+def load_specfile(specfile):
-    if len(licenses) == 0:
+    """Get licenses from the specfile content."""
-        file.write("License  : " + default_license + "\n")
+    global licenses
-    else:
+    global license_files
-        file.write("License  : " + " ".join(sorted(licenses)))
+    global hashes
-        file.write("\n")
+    specfile.licenses = licenses if licenses else [default_license]
+    specfile.license_files = sorted(license_files)
+    specfile.hashes = hashes

autospec/license_blacklist

@@ -0,0 +1,172 @@
+# This file contains the list of strings in the license blacklist
+# in the form: <blacklisted string>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+# For strings that start with '#', escape the '#' as '\#'.
+=========================
+%
+%license
+(LGPL)
+(new)
+(specified
++
+-
+-MIT
+-or-
+.git
+.md
+.mit
+.txt
+/
+1-2-3
+1.0
+1.1
+2.0
+2BSD
+3-Clause
+3-clause
+3BSD
+@CPACK_RPM_PACKAGE_LICENSE@
+AGPLv3+
+ALv2
+AND
+APL-2.0
+APPLICATION__TYPE
+ASL
+Artistic
+BSD
+BSD(3
+BSD-like
+BSD-style
+BSDL
+BSD_3_clause
+BSDish
+BSL
+CC-BY
+Commercial
+Corp.
+Distribution
+Domain
+Dual
+Dual GPL/BSD/CPL
+EPL
+Eclipse
+Expat(MIT/X11)
+Expat/MIT
+Foundation
+FreeBSD
+GENERAL
+GFDL
+GNU
+GPL
+GPL+
+GPL/BSD
+GPL/BSD/CPL
+GPLv2.1
+General
+IBM
+LESSER
+LGPL
+LGPL+BSD
+LGPL/MIT
+LICENCE
+Lesser
+Licences
+License
+License(2.0)
+License(==-2.0)
+License(>=-2.0)
+License,
+License-2
+License-2.0
+License-2.0(MPL-2.0)
+Licensing
+Lucent
+MIT/Expat
+MPL
+Modified
+Mozilla
+Muddy-MIT
+N/A
+New
+OFL
+OR
+Open
+PIL
+PSF-2+
+PUBLIC
+Public
+Revised
+SIL
+See
+Set
+Software
+Source
+Standard
+The
+This
+Two-clause
+UN
+UNKNOWN
+Unknown
+Unkown
+Unlimited
+Version
+Version-2.0
+What
+\#
+advertising
+and
+any
+bsd
+classifiers)
+clause)
+details
+domain
+dual
+exceptions
+for
+gpl
+http://nmap.org/man/man-legal.html
+is
+it
+later
+later(LGPLv2+)
+license
+ndg/httpsclient/LICENCE
+new
+none
+on
+open_source
+option
+option)
+or
+or(at
+others
+public
+style
+terms
+the
+under?
+unknown
+unrestricted
+uses
+using
+v1.0
+version
+version-2
+version-2.1
+with
+your
+|
+~
+Permission
+a
+charge
+copy
+granted
+hereby
+obtaining
+of
+person
+to

autospec/license_hashes

@@ -0,0 +1,288 @@
+# This file contains license hashes and the licenses they are associated with
+# in the form: <hash>, <license>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+0005480dce93b70f7d62fa311f49e3c6c1a6dcfa, MIT
+003da9a1871de68693740b39bee3ddca66707ad3, LGPL-2.1
+00bf1929edfbe209d33d4b58a47e0178280bd606, MIT
+01a6b4bf79aca9b556822601186afab86e8c4fbf, LGPL-2.1
+0292928b4571cd06a52307940df8bcffc4c51ea4, MIT
+0305317c0f694ba11e8f059938fd0c880356e7bc, BSD-3-Clause
+061048a4b794b6b63e2b81f93404a9eaf5f61f11, GPL  Ghostscript
+06877624ea5c77efe3b7e39b0f909eda6e25a4ec, GPL-2.0
+075d599585584bb0e4b526f5c40cb6b17e0da35a, GPL-2.0
+090586b9e4c51fd5ef3c39f25d2469a8be8e33c9, AFL-2.1 GPL-2.0+
+098e171b1c9c7f15e05e696a187328e3c9aa0099, BSD-4-Clause
+0b184ad51ba2a79e85d2288d5fcf8a1ea0481ea4, GPL-2.0
+0e8e850b0580fbaaa0872326cb1b8ad6adda9b0d, LGPL-2.0
+10782dd732f42f49918c839e8a5e2894c508b079, GPL-2.0
+11464e106e37066a94a23fe4912581799daa3e41, 3BSD
+11c746c0c250304a17169f4d6aa4a82d09fd7157, GPL-2.0
+12549569098cbb0efa8a4aa91f5f38068791fe42, GPL-2.0
+13d2034b5ee3cb8d1a076370cf8f0e344a5d0855, GPL-2.0
+14c1f9f1263a82d58615c51369e1443c3d9f15bf, LGPL-2.1
+14daa45932568feb416c5dd45f1cd909294fb3b0, MIT
+14e8b95e4871fb8a09ebbb5082d34a3c1c7da5a3, BSD MPL
+1665590806e57d72419af943374652cebc42571f, LGPL-2.1 MPL1.0
+174c509aa1452ab3a56faa01aec149015c391e99, LGPL-2.1+ GPL-2.0+
+175e59be229a5bedc6be93e958a970385bb04a62, Apache-2.0
+18b0624b094dd81f099af8672ff5a05259249059, GPL-2.0+ LGPL-2.1+
+196f0bcaf72257201b7e9d499a3c5b5f3f75c170, MIT
+1adfe59a1df9ae50ec572004dd5ebd5f45cc85f7, GPL-3.0+
+1b07b3f8b3b7d79c074d00dfb39a5be425e70cb8, GPL-2.0
+1c0c6888759a63c32bca7eb63353af2cd9bd5d9e, bzip2-1.0.6
+1c74b6200eb29fa8c10588f6e32f3f7cf8eda03f, MIT
+1cc9ab3464ce507485f4982670d12916ec5a3c0e, MIT
+1cf6c2c767c8f1d457c8156aafc9220dd9dedbe5, BSD
+1d6467e08284270291d138c7ea07f24689483068, BSD
+1e2bf763a406378034f0722de926e819c31ed211, GPL-3.0
+1f9b387f1c4c2857808696c5f2d7f9f31c26b911, OldMIT
+21e40770dfa802576ee515345cef47ecedff6524, LGPL-2.0+
+235350d09d078e0fb82939483642d7567b1d1245, MIT
+23a1e6369b12379f07c61bbbaa73e13704ebc30e, MIT
+24790001231306234389a547f69e43601471db75, BSD-3-Clause
+2504e714566874eaff0a4ff3ca44d354b135efe9, MIT
+253c30cd74e4812f13e9e561cb54cbab26bc19dc, GPL-2.0
+254aec9ad4aa42eb4f11c7a58ea1d297e14c43ae, MIT
+254f719409a77168a80aed51593017f6f44165c7, LGPL-2.1 MPL-1.1
+2807f3f1c4cb33b214defc4c7ab72f7e4e70a305, MIT
+2a18f0964f8f7f91a74026ffe45927c83441f333, 3BSD
+2a4fd3610e85e2efdefe2dc331b555794f9e0cb3, GPL-3.0
+2aa8684f0abf57011bd8bf5c30b0390cc2c85a03, MIT
+2aaa700763cac84c6ee8ce491a4148064850a761, Chinese something
+2b61cd7d9b22e98804387e896a3cfa382c1bc4ef, MIT
+2b8b815229aa8a61e483fb4ba0588b8b6c491890, Apache-2.0
+2b9d60c2972b476384af9900276837ac81954e80, GPL-2.0
+2c938b2fe439b3d78ca579e5fc5cf49b3218c0ab, MIT
+2cf9061e85ff400986a10c505a6177ff2b5e992d, MIT
+2d29c273fda30310211bbf6a24127d589be09b6c, GPL-2.0
+313e4788e0730c747b41fad603f0fba3765a022d, MIT
+31b3a6df67bb414219baa57025a5c8d6b79555a5, MIT
+329173a86dabff7ed1c8bd39117d410401c46a05, GPL-2.0+ LGPL-2.0+
+32b0302709046c7d107587b2a4435e91bcea2be8, MIT
+3348e5430ba4fb49fa8eb6e9caf4f06266639d0d, EPL-1.0
+3785dd882865259e99bf41ce033b122370e25ce1, MIT
+38abaf3c887f1799769b604e6d1ada4fcb81f8a0, MIT
+38f27835622344a460610effe7888c9c2ef40c74, MPL-1.0 LGPL-2.1
+39a21f33cadea18adcc23bf808d7d5ea6419c8b1, LGPL-2.1
+3b90aaf730fa20460f8fe3fd20c16daf3acaba59, MIT
+3c33e98325d414f0290c82bc32b0d8c28cbeeb9b, MIT
+3c53de6040bbb05b7ad825a48f8d7341f9e3d6f4, MIT
+3ee0019d4f4ea0a9d3f50800833f30dc14e2968e, GPL-2.0
+40a59387a5d3ab7449e77079f542f86018c16bff, GPL-2.0
+40f6d7c0dba74ddd10663dfa50c2659cbe251423, MIT
+430bd4bbe476b94eb686e5751395a242fc2354e0, BSD
+4386a89a4295085459ef1d54f3ab8859f1f1d28c, LGPL-2.1
+4406aeb00f12a2c9cfba8066eef1a7a0c9b32f43, 3BSD
+44f7289042b71631acac29b2f143330d2da2479e, LGPL-2.0
+46243828d8d29354fd01836ba3a12254c072eedc, GFDL-1.2
+47ca90dc423091f8db250f2cc45f28aba88d11ed, 2BSD
+480ff2fd1236e94295d10551b0b333296f639300, LGPL-2.1+
+48703751ffc7d48dc3aa9609be19dd542cf9776b, GPL-3.0
+4882eb6580148a083a7745785ef76768de52086c, MIT
+48d3aad525d9acd423ac6021c44fa4d15d4ee9ad, BSD-3-Clause
+49113556cdf35023603d71001aa073eadbd26b7e, BSD-3-Clause
+4aa399e286b0bfbfba4819711a9c143b79aa01d0, GPL-2.0
+4c7d9cdbd787a28d9d0d4e1a425cd1e826621e41, LGPL-2.1
+4cc77b90af91e615a64ae04893fdffa7939db84c, GPL-2.0
+4cc8868f0a2c423cc8d63fede50ae0fecd58a720, GPL-3.0+
+4d1d37f306ed270cda5b2741fac3abf0a7b012e5, GPL-2.0
+4d328180e5349a9f65193ed4f424300d0c2c6adc, GPL-2.0
+4df5d4b947cf4e63e675729dd3f168ba844483c7, LGPL-2.1
+4e791d47822f12167fcddffaa7b8231b0c3b300a, MIT
+4f35a76dbcc982e002982b97c5d93f3d95c5e57f, MIT
+4f485ab7059ac53d9e3818278ad82217ce976a36, GFDL-1.1
+4ff4760dfc54975ddb40a3455633574849bc5611, BSD-2-Clause
+50147984b983a89bb0a981968bad2e431f0a1423, MIT
+503df7650052cf38efde55e85f0fe363e59b9739, GPL-2.0
+5050f00b9e4605253a0a9d03d227bf0fb74fd676, MIT
+51ae38c0bc1c4f8b798d3181a3673ceebc93c54b, GPL-2.0
+5292f678d9f514464de238e26705c4e81c90366c, MIT Apache-2.0 BSD-3-Clause
+53afb4c86c46b4980f3f8bb129628c55e740226e, LGPL-2.0
+5405311284eab5ab51113f87c9bfac435c695bb9, GPL-2.0
+545f380fb332eb41236596500913ff8d582e3ead, LGPL-2.1
+570d185ea721e7d6aee7426be1b10a800af98aa8, LGPL-2.0
+57e93c186a48f2ccdd65f1872314534524825e94, MIT
+58e79a1175dc5c95f8ac7dc1d23f1ffc61a9fee7, WEIRD
+597bf5f9c0904bd6c48ac3a3527685818d11246d, LGPL-2.1
+5b550d122aa18a381e29e5be5f4814aeb7b6b800, MIT
+5b811940f3ef0c79e86f6ab11b0e4eced5e7eb14, BSD-3-Clause
+5bb165c9b21503806d60c4559af198f569dc46f6, LGPL-2.1
+5c96c2383498d529b9f62385b9db5ed6f290a890, MIT
+5d1a41800996f1c023b69f4b1c3a93b194a66e80, MIT
+5e7b36dfb18c7b8002bb9c41f87b65d280abd2ae, GFDL1.1
+5eb1e4cf540bfae563680ce53c71c261abb5f076, CERT weird
+5fb122a984b09d5c687513bb34a51eeeff2b13a7, BSD-3-Clause GPL-1.0
+5fb362ef1680e635fe5fb212b55eef4db9ead48f, LGPL-2.0
+61200da0f665a7b331f2f17b065b5a948fad40b8, MIT
+613732109bb85dacf28dccf37b7ec0c212124ecd, MIT
+618c9d05efe2932bc3aa4745ecfb548303dd8f08, MIT
+61bb7a8ea669080cfc9e7dbf37079eae70b535fb, MIT
+62066b13a35774834f104c2ff075ce679a0f75a3, EvolutionMix
+62ba6d3ef166a2b04cf4dcec7039c7afa9dcc2bc, MIT
+63b588642ebe669d50ab658ea1d079bb99086396, GPL-2.0
+654d5ed6dd2d6ab7904d4047cde6345730f9d174, GPL-3.0
+6596e03f9467e3d698993a49289a502c4baf7e08, MIT
+669dfab6308e814f23c524fd51f565a39536ec21, MIT
+66c77efd1cf9c70d4f982ea59487b2eeb6338e26, LGPL-2.1
+6700131aba37c25eb7f67b0c4071d02df6de75bc, MIT
+6723aef56fa8b33bec08e1477f4393e6707c973c, MIT
+6745330da3e7bde244b20b96a42eae659644e731, BSD-3-Clause
+68c94ffc34f8ad2d7bfae3f5a6b996409211c1b1, GPL-2.0
+6ae662f45f5458787a717d7280b6074d352de549, MIT
+6af05e5d7c8c49b139d589a27cddd8468e65a5a3, MIT
+6b1edb9ab8161370f1ba8686ec730eb2cc0c1a13, MIT
+6c394114eefb35de30b9675a99cc1500c6e574cb, MIT
+6c6ec2b7a40bad98fa4a5b55c3855258e5d2a05e, LGPL-2.1 GPL-2.0
+70c32f13c99ca9bb7d53d5cb35fa8243dfa30e76, MIT
+70e64fe9090c157e441681779e0f31aad34f35cb, GPL-3.0
+715a897b37b8e535c5812e5f4e663c30b5a67115, MIT
+72b40d717fc8757da66964a3a5b565aae69d8b39, GPL-3.0
+72d42b3c12e7765c1539d6741cbc2054c2d11260, GPL-3.0+
+72f59fbac43fa1a7f0607d7fdba747832e626656, 3BSD
+73ccddd0c27b4492f8182c35f0f7c434b998757c, GPL-2.0
+747f912a17321a200ab786514210c32116eb51b4, MIT
+74a8a6531a42e124df07ab5599aad63870fa0bd4, GPL-2.0
+74acefbff6c13e5c81492d7460b82e2b5ec16c1b, LGPL-2.1
+74cb2bfd42807a875d1386f2bae67be20c4e19cc, MIT
+750b9d9cc986bfc80b47c9672c48ca615cac0c87, BSD-3-Clause
+794a893e510ca5c15c9c97a609ce47b0df74fc1a, BSD-2-Clause
+79cd7284f964b96ba6df6c35dee5e8a92b9e9b49, 3BSD GPL-2.0+
+7b7e5d72495f3cd2b0d1307b28b92df4ad68a0a8, BSD-3-Clause
+7cdb2709d5ce063238a7fae991c5d6cbe3bc7d7f, LGPL-2.1Apache2
+7f3f67aef48ead049bebdab307c04c2e03342710, 3BSD
+7f673c53e3a7840abd45c5869b6ab8aeb2d34c10, MIT
+8088b44375ef05202c0fca4e9e82d47591563609, LGPL-2.1
+80a39eb9544a657a0e23f53a15daff8a1d6a0e7d, LGPL-2.1
+8244c6af44d86382af14ef78dc6f430925cb46a7, Perl
+8313e316436df0f02ce407f0b52b77f8195bfae6, PublicDomain
+83b927c3fa44af01d2515ea8575f8d4848cc10ec, GPL-2.0+ LGPL-2.0+
+83ba6546e00f890f3a26a9bedd264084f8527d5e, LGPL-2.0
+842745cb706f8f2126506f544492f7a80dbe29b3, GPL-3.0
+84d1b5be6b879fb3967c937a581292206a68ab45, MIT
+8570a5c766d8b7eda6fa171a7f6674e335b50c5e, MIT
+8624bcdae55baeef00cd11d5dfcfa60f68710a02, GPL-3.0
+87d59feee73204c7334b7e00ae1cec3f3c583474, MIT
+88aacfcae9d42c894660950b01e7229cc2895cad, MIT
+893c940eb56cd6a45620c65c72b9a8c48bd79217, MIT
+89e91cab64960b7c852e3b31da3802029af6ef88, GPL-2.0
+8e57ffebd0ed4417edc22e3f404ea3664d7fed27, MIT
+8edc79a3e927a60794fbccaaa22835f9e1ef5c9a, GPL-3.0+
+900806db6414f1bb309ab7438c0a5bac52eb3c2b, CCSA3.0
+904c53ec5a9edfd5d08fcef5fb8ff9c844aed213, MIT
+909b58c9b803acb8d063ac6b2147e56afc8055f6, GPL-2.0
+922e3ebe82d80cae2f6c89bbb065ae455d0265b6, MIT
+937157040d49ab4bbcf2070ed8f9b6849fdfa442, MIT
+9384394a530b4a93530da89f95e05ac86c4880a0, MIT
+943fb830004a8813177c761482ac5d583ee09351, GPL-3.0
+94e50e74aa533e042a8fb41a0d5b07460cdc3b91, GPL-2.0
+952348c4126a5946c98385d2fe5a4801735f174d, MIT
+96784c94078c65a3af7fe6e26b60448042aea27a, IPL-1.0
+97cf1f1fdaaff217f90d58faa9d1b2c0e387fd96, MIT
+99b5245b4714b9b89e7584bfc88da64e2d315b81, BSD
+9a1929f4700d2407c70b507b3b2aaf6226a9543c, LGPL-2.1
+9dc4c58e46864b14d95a1368a6ebac37aad89d32, BSD-3-Clause
+9e010b11705daec7ebf9c01a32d6f09ee444968d, GPL-2.0+
+a08228c5c15d51b7072c406b0d30d862af339ada, MIT
+a280389ff9a8a9d58c76b406ac8a9b64d679f587, MIT
+a2f64f2a85f5fd34bda8eb713c3aad008adbb589, MIT
+a4777b6caf11b08abc996727d094e2dafc16caaa, MIT
+a5771f371f97e15ece15ad91386b8897cb06a95a, 3BSD
+a6adc13d0c809ab8cb68e6e3b6eb7571bd0e2920, GPL-3.0
+a7a897a4bde987e597c04f16a9c28f6d3f57916d, GPL-2.0
+a7c968e282631e3b29c66bcf4a438fc72b68be27, MIT
+a9fbc5ee6bc3b991c1fee735b96204001a8d64fe, SGI-B
+aa4e6280a7b029d15f5be03d2858e5e4c1655ac8, BSD-2-Clause
+aade406c56b751113933f3b64890fb8d5b489a2c, LGPL-3.0
+aba8d76d0af67d57da3c3c321caa59f3d242386b, MPL-1.1
+adcc6ae0ba0e1e4fae77bd2b98db0d5a1aae2e75, BSD-2-Clause
+b034fd3ba225f77f2b5853bcc65393b50761dfd7, MIT
+b0fbb2162c8af440cb60a412a145a01195644b3b, 3BSD
+b256632dcce76559734ff0a23330d2898b7d3a3b, GPL-2.0
+b307e9607ffa7ab6dc4ab7c019d0e62a627b9532, MIT
+b37df50521fc00c23c1835b592967fcce90cd23b, MIT
+b47456e2c1f38c40346ff00db976a2badf36b5e3, GPL-2.0
+b55b6493b790dc1291794a6b4f04064ce06bf435, MIT
+b6e9e05950ebcd16852fe9795b564a3f5d976223, Unknown
+b72fac76415d8029e2823485e3fb9ddb650b879e, GPL-2.0
+b7c6cc817dedc6e9c0aff3bb5a50cbc1e7f9548f, MIT
+b7e55cb48a9fb891583a221fa909ccd75e2ceaaf, LGPL-2.0
+b972655523ae3fdb35306d830b8426e4af30d29c, GPL-2.0+
+b9c8da2c5e4162fc7e321fda0b8d3d084681e764, 2BSD
+ba8966e2473a9969bdcab3dc82274c817cfd98a1, LGPL-2.0
+ba905468a032e17d551b8185bccecea5807b7638, GPL-3.0
+bb306f5409529d1c19b78d60f705d37e3c470124, MIT
+bb38ed62633b427973e8bcbe95c88fc1e00fd77d, MIT
+bb40dd090e18e858168d53bf519129e990f156dc, MIT
+bb5cc149358d8c36484f012e24ce694f66bd3b12, FONT
+bbdf8e1d71045a407fee797ad27983e3eafa3ee6, MIT
+bc06cbdf781c87d2df2fe385214f936d010dd2a2, 3BSD
+bc252631805cf037048f64fef562f98c2a0bdc9e, 3BSD
+bd67b052cf8bd1c31dbcdbe56955b6b5d22243f7, MIT
+be0b40ce8f9532b75966a20d14af123d3c6b05aa, GPL-2.0
+be90ae29e7e0977910b397571efe56601c123793, MIT
+bf50bac24e7ec325dbb09c6b6c4dcc88a7d79e8f, LGPL-2.0
+c2e8ebe3551b42d4a868922ec922a8af6452c48b, LGPL-3.0
+c3795529e138884da8f5b836db7e2fe1025526cd, GPL-2.0
+c3de6dd37753714ce85482acd8292525d4515595, GPL-2.0
+c5041828c23a816cc38e662b61032e959cb86893, LGPL-2.1
+c540cc53356ba854b2673e53fabc05b2f579404c, GPL-2.0
+c61905dc64311e8bcee6afc425fa40f917a45131, GFDL1.1
+c89f43c4122e411a59ba705cfcf89a3f88019fcf, 3BSD
+c931aad3017d975b7f20666cde0953234a9efde3, GPL-2.0
+caeb68c46fa36651acf592771d09de7937926bb3, LGPL-2.1
+cb03b5ef1e3d983356601be29e306eebdc6a9257, MIT
+cb9f6ee72658f9cb9fceb8b4a3d3e280a4586f20, LGPL-2.1
+cbd3c98acb262700c79d534692dc08cf05ec7f00, MIT
+cbfdfb0078ac466712aa0a82242b8fc955b0bc00, MIT
+cc1e4801215dbc785fbc3d2b9910b2f093466041, GPL-2.0
+cc4023c25020d429d9f4a4adf1a2461c8f334597, 4BSD
+ce154c6a0d76d90cd20ef010636eb8e98a4f9fe5, MIT
+ce714346889d1becdd3132cf7380c734a985ecf1, GPL-1.0 Perl
+cf0f32a176b91013767b8a4a727a369156c6558c, MIT
+cf3eaf29116a37a7d9ba773e776104c067c8e5fc, MIT
+cfea9b772a9da707b5f092d3c18b4572fa4bd520, LGPL-2.0
+d004d8de0c32dfd3e86ca5f4a4552332bf5bbb67, GPL-2.0 LGPL-3.0 APL-2.0 GFDL-1.3
+d0bee23991ce69abdc36d8711f54982ca2fd615c, MIT
+d129361510b2cbbf83a0c2d3dc0381b76d87ac9a, LGPL-2.1 MPL-1.1
+d13c7d638806dc37e5c9b6bbc39366cd904d04de, MIT
+d1845ff7fcb2cdc1f5713dd18140e89de2ab0204, MIT
+d25eb6311e5d0398a98818545298bc0fa10efab1, GPL-2.0
+d46d26c49b071f6f17913af46023b538bf1768e7, MIT
+d4e89d1a1e7812dae053aa8cb891f452891df932, GPL-2.0
+d6dad56b15f36205dc3c605a3e5a98ccac1275f2, MIT
+d83b6378d06fdf228b1afc0bf97e09b44bbb2e7b, LGPL-2.0
+d86fb3f981d36e1f2890c7aaea6f8bcbb7ad4a27, Muddy-MIT
+d8b8b6fadbff8363d66d5c703971ab0c86696e26, MIT
+dcc7cc194ef7fcd3e5f8e9bc20ddad371ee1d36c, MIT
+de9f614d7db67dd479cfa87b10a57ecc4abc7295, MIT
+deed0c8eb4eb962dc74006326c98f750e08c8e3f, 2BSD 3BSD LGPL
+df734091d6dbc9d20382b4084e68396a483582bd, 2BSD Ruby
+dfac199a7539a404407098a2541b9482279f690d, GPL-2.0
+e00070d4eca0371dd3ca3df4a2472cc2db599bad, GPL-2.0
+e16364503a826791c0588c304c794ee8d210b7bc, GPL-1.0
+e1d31e42d2a477d6def889000aa8ffc251f2354c, GFDL-1.3
+e2f32873edf33dd9f03272bb5aa47f825369b905, MIT
+e3afbcad74dd08819b5de5ad0338203369731f67, GPL-2.0
+e4a57e19916359bf6f6f36847712b1761204feed, BSD-3-Clause
+e4b3a39cf858dd93c1d5766b8bf1b49fbe9a1d67, BSD-3-Clause
+e60c2e780886f95df9c9ee36992b8edabec00bcc, LGPL-2.1
+e7d563f52bf5295e6dba1d67ac23e9f6a160fab9, GPL-3.0
+e895154f04fa9aa52d1742728d984428f26a764b, LIBPNG
+e911adf5641a09f13fdd5d59962ad37da043df79, MIT
+ea754f5e8f103855a5574468c9101889553c917e, MIT
+edce7bb3398021ac27115ab50d8db9efaf3408c7, LGPL-2.1
+ef51e850423393335a21c5069af73674cdf6753f, GPL-2.0
+f0236e5d5e53eb7b386cd3ac80b852814e3112dc, GPL-2.0 LGPL-2.1
+f33b4d7ca167851483857c947ea7484d533d9b05, GFDL CCBYSA
+f35b1294fe252bc8ec243713fe9e214bbe5c6069, MIT
+f45ee1c765646813b442ca58de72e20a64a7ddba, GPL-3.0
+f4c513b3be30f40bb34a6e4d8c4f45a5bf1b6a10, Meta
+f6e0bb551b32440d84c3886a34ddbd3d1aec7b56, Apache2.0
+f7e86954cf5438549ec8948212f2713024119c68, GPL-1.0+ Artistic
+f9c9a2d3495a0766b4cf20d4b90cfe714dab3dc1, MIT
+f9e3dfab1d42d7282b0654e921706147795be7cb, LGPL-2.1 GPL-2.0+
+faa78032a41c2bfe6bfd1df72ccd1bb7ab4f7a10, GPL-2.0
+fbf1ca3836c050241ed2fde6ac3a515313f30e8b, GPL-2.0
+fd02365360c1936fdd069b08d29c16c6b16e4ea4, MIT
+fddb19e45cceea1a6c3a9b16547b824bb768eceb, MIT

autospec/license_translations

@@ -0,0 +1,104 @@
+# This file contains license strings and the actual licenses they are associated
+# with in the form: <license string>: <license>
+# This file is sorted with LC_COLLATE=C
+# Lines beginning with '#' are ignored.
+# For strings that start with '#', escape the '#' as '\#'.
+2-clause, BSD-2-Clause
+AGPL-3, AGPL-3.0
+APL-2.0, Apache-2.0
+APL2, Apache-2.0
+APL2.0, Apache-2.0
+ASL 2.0, Apache-2.0
+ASL-2, Apache-2.0
+ASL-2.0, Apache-2.0
+Apache 2.0, Apache-2.0
+Apache License 2.0, Apache-2.0
+Apache License, Version 2.0, Apache-2.0
+Apache, Apache-2.0
+Apache-2, Apache-2.0
+Apache2.0, Apache-2.0
+Apachev2, Apache-2.0
+Artistic-1.0+GPL-1.0, Artistic-1.0 GPL-1.0
+Artistic_2, Artistic-2.0
+BSD(3-clause), BSD-3-Clause
+BSD-2-clause, BSD-2-Clause
+BSD-3-clause, BSD-3-Clause
+BSD_2_clause, BSD-2-Clause
+BSD_3_clause, BSD-3-Clause
+Boost, BSL-1.0
+CC0, CC0-1.0
+CPL, CPL-1.0
+Common Public License Version 1.0, CPL-1.0
+Expat, MIT
+GFDL1.1, GFDL-1.1
+GPL (> 2), GPL-3.0+
+GPL(==-2), GPL-2.0
+GPL(>=-2), GPL-2.0+
+GPL(>=-2.0), GPL-2.0+
+GPL(>=-2.1), GPL-2.0
+GPL(>=-3), GPL-3.0
+GPL(>=-3.0), GPL-3.0+
+GPL(>=2), GPL-2.0+
+GPL(>=3), GPL-3.0+
+GPL-2+, GPL-2.0+
+GPL-2, GPL-2.0
+GPL-2.0+, GPL-2.0+
+GPL-2.0+LGPL-2.1, GPL-2.0 LGPL-2.1
+GPL-2.0, GPL-2.0
+GPL-2.0-or-later, GPL-2.0+
+GPL-3+, GPL-3.0
+GPL-3, GPL-3.0
+GPL-3.0+, GPL-3.0+
+GPL-3.0, GPL-3.0
+GPL2, GPL-2.0
+GPL3, GPL-3.0
+GPLV2, GPL-2.0
+GPLV3, GPL-3.0
+GPLv2+, GPL-2.0+
+GPLv2, GPL-2.0
+GPLv3+, GPL-3.0+
+GPLv3, GPL-3.0
+ISCL, ISC
+LGPL(>=-2), LGPL-2.0+
+LGPL(>=-2.1), LGPL-2.1+
+LGPL(>=-3), LGPL-3.0+
+LGPL(>=2), LGPL-2.0+
+LGPL-2, LGPL-2.0
+LGPL-2.0+, LGPL-2.0+
+LGPL-2.1+, LGPL-2.1+
+LGPL-2.1-or-later, LGPL-2.1+
+LGPL-3+, LGPL-3.0+
+LGPL-3, LGPL-3.0
+LGPLv2+, LGPL-2.0+
+LGPLv2, LGPL-2.0
+LGPLv2.1+, LGPL-2.1+
+LGPLv2.1, LGPL-2.1
+LGPLv3+, LGPL-3.0+
+LGPLv3, LGPL-3.0
+MIT/X, MIT
+MPL-2, MPL-2.0
+MPL2, MPL-2.0
+MPLv1.1, MPL-1.1
+MPLv2, MPL-2.0
+MPLv2.0, MPL-2.0
+MPLv2.0,, MPL-2.0
+PSF, Python-2.0
+Perl, Artistic-1.0-Perl
+Python, Python-2.0
+VIM, Vim
+ZLIB, Zlib
+ZPL 2.1, ZPL-2.1
+ZPL, ZPL-2.0
+apache, Apache-2.0
+artistic2, Artistic-2.0
+artistic_2, Artistic-2.0
+gplv3, GPL-3.0
+http://creativecommons.org/licenses/BSD/, BSD-2-Clause
+http://opensource.org/licenses/MIT, MIT
+http://www.apache.org/licenses/LICENSE-2.0, Apache-2.0
+lgpl, LGPL-2.1
+mit, MIT
+perl, Artistic-1.0-Perl
+w3c, W3C
+zlib, Zlib
+zlib/libpng, zlib-acknowledgement

autospec/logcheck.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+#
+# logcheck.py - part of autospec
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import sys
+
+from util import print_fatal, write_out
+
+
+def log_etc(lines):
+    """Return the content of the START/etc ... END/etc section."""
+    etc = []
+    while True:
+        line = next(lines)
+        line = line.strip()
+        if line == 'END/etc':
+            break
+        if line.startswith('+'):
+            continue
+        etc.append(line)
+    return etc
+
+
+def logcheck(pkg_loc):
+    """Try to discover configuration options that were automatically switched off."""
+    log = os.path.join(pkg_loc, 'results', 'build.log')
+    if not os.path.exists(log):
+        print('build log is missing, unable to perform logcheck.')
+        return
+
+    whitelist = []
+    file_dir = os.path.dirname(os.path.abspath(__file__))
+    file_path = os.path.join(file_dir, 'configure_whitelist')
+    with open(file_path, "r") as whitelistf:
+        for line in whitelistf:
+            if line.startswith("#"):
+                continue
+            whitelist.append(line.rstrip())
+
+    blacklist = []
+    file_dir = os.path.dirname(os.path.abspath(__file__))
+    file_path = os.path.join(file_dir, 'configure_blacklist')
+    with open(file_path, "r") as blacklistf:
+        for line in blacklistf:
+            if line.startswith("#"):
+                continue
+            blacklist.append(line.rstrip())
+
+    with open(log, 'r') as logf:
+        lines = logf.readlines()
+
+    pat = re.compile(r"^checking (?:for )?(.*?)\.\.\. no")
+    misses = []
+    iter_lines = iter(lines)
+    for line in iter_lines:
+        if line.strip() == "START/etc":
+            etc = log_etc(iter_lines)
+            if etc:
+                write_log(pkg_loc, "etc_files", etc)
+        match = None
+        m = pat.search(line)
+        if m:
+            match = m.group(1)
+
+        if "none required" in line:
+            match = None
+
+        if "warning: format not a string literal" in line:
+            match = line
+
+        if not match or match in whitelist:
+            continue
+
+        if match in blacklist:
+            print_fatal("Blacklisted configure-miss is forbidden: " + match)
+            misses.append("Blacklisted configure-miss is forbidden: " + match)
+            write_log(pkg_loc, 'configure_misses', misses)
+            sys.exit(1)
+
+        print("Configure miss: " + match)
+        misses.append("Configure miss: " + match)
+
+    if not misses:
+        return
+
+    write_log(pkg_loc, 'configure_misses', misses)
+
+
+def write_log(pkg_loc, fname, content):
+    """Create log file with content."""
+    write_out(os.path.join(pkg_loc, fname), '\n'.join(sorted(content)))

autospec/patches.py

@@ -1,44 +0,0 @@
-#!/bin/true
-#
-# patches.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Parse config files
-#
-
-
-patches = []
-autoreconf = False
-
-
-def write_patch_header(file):
-    counter = 1
-    for p in patches:
-        file.write("Patch" + str(counter) + ": " + p.split()[0] + "\n")
-        counter = counter + 1
-
-
-def apply_patches(file):
-    counter = 1
-    for p in patches:
-        name = p.split(None, 1)[0]
-        if name == p:
-            options = "-p1"
-        else:
-            options = p.split(None, 1)[1]
-        if not p.split()[0].endswith(".nopatch"):
-            file.write("%%patch%d %s\n" % (counter, options))
-        counter = counter + 1

autospec/pkg_integrity.py

@@ -0,0 +1,806 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+import download
+import util
+
+GPG_CLI = False
+DESCRIPTION = "Performs package signature verification for packages signed with\
+gpg."
+USAGE = """
+Verify package signature when public key present in default keyring:
+{fn} --sig package.tar.gz.asc --tar package.tar.gz
+
+Verify package signature when public key is provided as cleartext
+{fn} --sig package.tar.gz.asc --tar package.tar.gz --pubkey package_author\
+.pubkey
+
+Verify package signature when public key is in a keyring different from \
+default keyring
+{fn} --sig package.tar.gs.asc --tar package.tar.gz --gnupghome /opt/pki/gpghome
+
+Verify package signature when public key is provided as a file and keyring is \
+different from default
+{fn} --sig package.tar.gs.asc --tar package.tar.gs --pubkey package_author.\
+pubkey --gnupghome /opt/pki/gpghome
+
+""".format(fn=__file__)
+
+SEPT = "-------------------------------------------------------------------------------"
+KEYID_TRY = ""
+KEYID = ""
+IMPORTED = ""
+EMAIL = ""
+GNUPGCONF = """keyserver keys.gnupg.net"""
+CMD_TIMEOUT = 20
+ENV = os.environ
+INPUT_GETTER_TIMEOUT = 60
+CHUNK_SIZE = 2056
+
+KEY_CACHE_DIR = os.path.expanduser('~/.cache/clr-pkg-key-cache')
+
+
+def update_gpg_conf(proxy_value):
+    """Set GNUPGCONF with http_proxy value."""
+    global GNUPGCONF
+    GNUPGCONF = "{}\nkeyserver-options http-proxy={}".format(GNUPGCONF, proxy_value)
+
+
+if 'http_proxy' in ENV.keys():
+    update_gpg_conf(ENV.get('http_proxy'))
+elif 'HTTP_PROXY' in ENV.keys():
+    update_gpg_conf(ENV.get('HTTP_PROXY'))
+
+
+# CLI interface to gpg command
+class GPGCliStatus(object):
+    """Mock gpgmeerror."""
+
+    def __init__(self, strerror):
+        """Initialize mock GPGCliStatus."""
+        self.strerror = strerror
+
+
+class GPGCli(object):
+    """CLI wrapper for gpg."""
+
+    @staticmethod
+    def exec_cmd(args):
+        """Popen wrapper."""
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        try:
+            out, err = proc.communicate(timeout=CMD_TIMEOUT)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            out, err = proc.communicate()
+        return out, err, proc.returncode
+
+    def __init__(self, pubkey=None, home=None):
+        """Set GPGCli defaults."""
+        _gpghome = home
+        if _gpghome is None:
+            _gpghome = tempfile.mkdtemp(prefix='tmp.gpghome')
+        os.environ['GNUPGHOME'] = _gpghome
+        self.args = ['gpg', '--homedir', _gpghome]
+        util.write_out(os.path.join(_gpghome, 'gpg.conf'), GNUPGCONF)
+        if pubkey is not None:
+            args = self.args + ['--import', pubkey]
+            output, err, code = self.exec_cmd(args)
+            if code == -9:
+                raise Exception('Command {} timeout after {} seconds'.format(' '.join(args), CMD_TIMEOUT))
+            elif code != 0:
+                raise Exception(err.decode('utf-8'))
+        self._home = _gpghome
+
+    def verify(self, _, tarfile, signature):
+        """Validate tarfile with signature."""
+        # Since autospec can only verify one signature for now, extract the
+        # first signature from the detached signature file.
+        sig_name = signature
+        packets = parse_gpg_packets(signature)
+        if len(packets) > 1:
+            # sig file may be ascii-armored, so dearmor it first...
+            args = self.args + ['--dearmor', '--output', '-', signature]
+            output, err, code = self.exec_cmd(args)
+            if code != 0:
+                return GPGCliStatus(f'Failed to convert {signature} to binary format')
+            num_bytes = packets[0].get("length")
+            if not num_bytes:
+                return GPGCliStatus(f'Cannot verify first signature from {signature}')
+            first_sig = output[:num_bytes]
+            with tempfile.NamedTemporaryFile(prefix="newsig-", dir=self._home, delete=False) as new_sig_file:
+                new_sig_file.write(first_sig)
+                sig_name = new_sig_file.name
+        args = self.args + ['--verify', sig_name, tarfile]
+        output, err, code = self.exec_cmd(args)
+        if code == 0:
+            return None
+        elif code == -9:
+            return GPGCliStatus('Command {} timeout after {} seconds'.format(' '.join(args), CMD_TIMEOUT))
+        return GPGCliStatus(err.decode('utf-8'))
+
+    def import_key(self, keyid):
+        """Import signer key."""
+        args = self.args + ['--recv-keys', keyid]
+        output, err, code = self.exec_cmd(args)
+        if code == 0:
+            return None, output
+        elif code == -9:
+            return GPGCliStatus('Import key timeout after {} seconds, make sure keystore is reachable'.format(CMD_TIMEOUT)), None
+        return GPGCliStatus(err.decode('utf-8')), None
+
+    def export_key(self, keyid):
+        """Export signer key with armor."""
+        args = self.args + ['--armor', '--export', keyid]
+        output, err, code = self.exec_cmd(args)
+        if output.decode('utf-8') == '':
+            return GPGCliStatus(err.decode('utf-8')), None
+        if code == 0:
+            return None, output.decode('utf-8')
+        return GPGCliStatus(err.decode('utf-8')), None
+
+    def display_keyinfo(self, keyfile):
+        """Show signer key information."""
+        args = self.args + ['--list-packet', keyfile]
+        lp, err, code = self.exec_cmd(args)
+        if code != 0:
+            return GPGCliStatus(err.decode('utf-8')), None
+
+        # trim list-packet output to 10 lines
+        lp = "--list-packet:\n" + "\n".join(lp.decode("utf-8").split("\n")[:10])
+
+        args = self.args + ['--fingerprint', os.path.basename(keyfile.replace(".pkey", ""))]
+        fp, err, code = self.exec_cmd(args)
+        if code != 0:
+            return GPGCliStatus(err.decode('utf-8')), None
+
+        fp = "--fingerprint:\n" + fp.decode("utf-8")
+        return None, "{}\n\n{}".format(lp, fp)
+
+
+@contextmanager
+def cli_gpg_ctx(pubkey=None):
+    """Return correctly initialized GPGCli."""
+    _gpghome = None
+    try:
+        _gpghome = tempfile.mkdtemp(prefix='tmp.gpghome')
+        yield GPGCli(pubkey, _gpghome)
+    finally:
+        if _gpghome is not None:
+            _ = subprocess.run(["gpgconf", "--homedir", _gpghome, "--kill", "all"],
+                               stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            del os.environ['GNUPGHOME']
+            shutil.rmtree(_gpghome, ignore_errors=True)
+
+
+# Use gpg command line
+def verify_cli(pubkey, tarball, signature):
+    """Validate tarfile with signature."""
+    with cli_gpg_ctx(pubkey) as ctx:
+        return ctx.verify(pubkey, tarball, signature)
+    raise Exception('Verification did not take place using cli')
+
+
+class Verifier(object):
+    """Base validation class."""
+
+    def __init__(self, **kwargs):
+        """Set default values."""
+        self.url = kwargs.get('url', None)
+        self.package_sign_path = kwargs.get('package_sign_path', None)
+        self.config = kwargs.get('config', None)
+        print(SEPT)
+
+    @staticmethod
+    def quit():
+        """Stop verification."""
+        print('Critical error quitting...')
+        print(SEPT)
+        exit(1)
+
+    @staticmethod
+    def calc_sum(filepath, digest_algo):
+        """Use digest_algo to calculate block sum of a file."""
+        BLOCK_SIZE = 4096
+        with open(filepath, 'rb') as fp:
+            digest = digest_algo()
+            for block in iter(lambda: fp.read(BLOCK_SIZE), b''):
+                digest.update(block)
+            return digest.hexdigest()
+
+    def print_result(self, result, err_msg=''):
+        """Display verification results."""
+        global EMAIL
+        package_name = ''
+        if self.url is not None:
+            package_name = os.path.basename(self.url)
+        if result:
+            msg = "{} verification was successful ({})".format(package_name, EMAIL)
+            util.print_success(msg)
+        else:
+            msg = "{} verification failed {}".format(package_name, err_msg)
+            util.print_error(msg)
+
+    def __del__(self):
+        """Display partition."""
+        print(SEPT)
+
+
+def get_signature_file(package_url, package_path):
+    """Attempt to build signature file URL and download it."""
+    sign_urls = []
+    netloc = urlparse(package_url).netloc
+    if 'samba.org' in netloc:
+        sign_urls.append(package_url + '.asc')
+    elif 'mirrors.kernel.org' in netloc:
+        sign_urls.append(package_url + '.sig')
+    else:
+        iter = (package_url + "." + ext for ext in ("asc", "sig", "sign"))
+        for sign_url in iter:
+            sign_urls.append(sign_url)
+
+    sign_file = None
+    dest = None
+    for url in sign_urls:
+        dest = os.path.join(package_path, os.path.basename(url))
+        sign_file = download.do_curl(url, dest)
+        if sign_file is not None:
+            return sign_file
+
+    return None
+
+
+def compare_keys(newkey, oldkey):
+    """Key comparison to check against key tampering."""
+    if newkey != oldkey:
+        util.print_error('Public key has changed:\n'
+                         '            old key: {}\n'
+                         '            new key: {}\n'
+                         'this is a critical security error, quitting...'
+                         .format(oldkey, newkey))
+        exit(1)
+
+
+# sha256sum Verifier
+class ShaSumVerifier(Verifier):
+    """Extend verification for sha sums."""
+
+    def __init__(self, **kwargs):
+        """Add shalen initialization."""
+        Verifier.__init__(self, **kwargs)
+        self.package_path = kwargs.get('package_path', None)
+        self.shalen = kwargs.get('shalen', 256)
+
+    def verify_sum(self, shasum):
+        """Verify sha sum."""
+        util.print_info("Verifying sha{}sum digest".format(self.shalen))
+        if shasum is None:
+            self.print_result(False, err_msg='Verification requires shasum')
+            return None
+        if os.path.exists(self.package_path) is False:
+            self.print_result(False, err_msg='{} not found'.format(self.package_path))
+            return None
+
+        sha_algo = {
+            256: hashlib.sha256
+        }.get(self.shalen, None)
+
+        if sha_algo is None:
+            self.print_result(False, err_msg='sha{} algorithm not found'.format(self.shalen))
+            return None
+
+        digest = self.calc_sum(self.package_path, sha_algo)
+        self.print_result(digest == shasum)
+        return digest == shasum
+
+
+# MD5 Verifier
+class MD5Verifier(Verifier):
+    """Extend verification for MD5."""
+
+    def __init__(self, **kwargs):
+        """Add MD5 initialization."""
+        Verifier.__init__(self, **kwargs)
+        self.package_path = kwargs.get('package_path', None)
+        self.md5_digest = kwargs.get('md5_digest', None)
+
+    def verify_md5(self):
+        """Verify MD5."""
+        util.print_info("Verifying MD5 digest")
+        if self.md5_digest is None:
+            self.print_result(False, err_msg='Verification requires a md5_digest')
+            return None
+        if os.path.exists(self.package_path) is False:
+            self.print_result(False, err_msg='{} not found'.format(self.package_path))
+            return None
+        md5_digest = self.calc_sum(self.package_path, hashlib.md5)
+        self.print_result(md5_digest == self.md5_digest)
+        return md5_digest == self.md5_digest
+
+
+# gnome.org Verifier
+class GnomeOrgVerifier(ShaSumVerifier):
+    """Verify sha sums for gnome.org."""
+
+    def __init__(self, **kwargs):
+        """Initialize with gnome.org url."""
+        kwargs.update({'shalen': 256})
+        self.package_url = kwargs.get('url', None)
+        ShaSumVerifier.__init__(self, **kwargs)
+
+    @staticmethod
+    def fetch_shasum(shasum_url):
+        """Get shasum file from gnome.org."""
+        data = download.do_curl(shasum_url)
+        if data:
+            return data.getvalue().decode('utf-8')
+        else:
+            return None
+
+    @staticmethod
+    def get_shasum(package_url):
+        """Try and get sha url based on package url."""
+        ext_pos = package_url.find(".tar.")
+        if ext_pos < 0:
+            shasum_url = "{}.sha256sum".format(package_url)
+        else:
+            shasum_url = package_url[:ext_pos] + ".sha256sum"
+        shasum = GnomeOrgVerifier.fetch_shasum(shasum_url)
+        if shasum:
+            return shasum
+        return None
+
+    @staticmethod
+    def parse_shasum(package_url, shasum_text):
+        """Parse shasum from sha url file content."""
+        for line in shasum_text.split('\n'):
+            if not line.strip():
+                continue
+            sha, file = [col for col in line.split() if col != '']
+            if os.path.basename(package_url) == file:
+                return sha
+        return None
+
+    def verify(self):
+        """Verify tar file with sha from gnome.org."""
+        if self.package_url is None:
+            self.print_result(False, err_msg='Package URL can not be None for GnomeOrgVerifier')
+            return None
+        shasum = self.get_shasum(self.package_url)
+        if shasum is None:
+            self.print_result(False, err_msg='Unable to find shasum URL for {}'.format(self.package_url))
+            return None
+        shasum = self.parse_shasum(self.package_url, shasum)
+        if shasum is None:
+            self.print_result(False, err_msg='Unable to parse shasum {}'.format(shasum))
+            return None
+        return self.verify_sum(shasum)
+
+
+# download.qt.io Verifier
+class QtIoVerifier(ShaSumVerifier):
+    """Verify sha256 hashes for download.qt.io."""
+
+    def __init__(self, **kwargs):
+        """Initialize with package URL."""
+        kwargs.update({'shalen': 256})
+        self.package_url = kwargs.get('url', None)
+        ShaSumVerifier.__init__(self, **kwargs)
+
+    def fetch_shasum(self):
+        """Fetch sha256 file associated with the package URL."""
+        shasum_url = "{}.sha256".format(self.package_url)
+        data = download.do_curl(shasum_url)
+        if data:
+            return data.getvalue().decode('utf-8')
+        else:
+            return None
+
+    def parse_shasum(self, content):
+        """Parse sha256 file."""
+        basename = os.path.basename(self.package_url)
+        line = content.split('\n')[0]
+        match = re.match(r'([0-9a-f]{64})\s+' + basename, line)
+        if not match:
+            return None
+        return match.group(1)
+
+    def verify(self):
+        """Verify the source file's sha256 hash."""
+        if self.package_url is None:
+            self.print_result(False, err_msg='Package URL is empty')
+            return None
+        content = self.fetch_shasum()
+        if content is None:
+            self.print_result(False, err_msg='Unable to download sha256 for {}'.format(self.package_url))
+            return None
+        shasum = self.parse_shasum(content)
+        if shasum is None:
+            self.print_result(False, err_msg='Unable to parse sha256 for {}'.format(self.package_url))
+            return None
+        return self.verify_sum(shasum)
+
+
+# GPG Verification
+class GPGVerifier(Verifier):
+    """Verify GPG signature."""
+
+    def __init__(self, **kwargs):
+        """Initialize gpg signature validation."""
+        Verifier.__init__(self, **kwargs)
+        self.key_url = kwargs.get('key_url', None)
+        self.package_path = kwargs.get('package_path', None)
+        self.package_check = kwargs.get('package_check', None)
+        self.interactive = kwargs.get('interactive', False)
+        if not self.key_url and self.package_check:
+            # signature exists locally, don't try to download self.url
+            self.key_url = self.url.rstrip('/') + get_file_ext(self.package_check)
+        if not self.package_sign_path:
+            # the key exists (or will exist) at
+            # <package directory>/<key url basename>
+            self.package_sign_path = os.path.join(os.path.dirname(self.package_path),
+                                                  os.path.basename(self.key_url))
+        # pubkey path is the package directory - this is where imports will go
+        self.pubkey_path = os.path.join(os.path.dirname(self.package_path), "{}.pkey")
+
+    def get_sign(self):
+        """Attempt to download gpg signature file."""
+        sign_file = download.do_curl(self.key_url, self.package_sign_path)
+        if sign_file is not None:
+            return True
+        else:
+            msg = "Unable to download file {}"
+            self.print_result(False, msg.format(self.key_url))
+
+    def verify(self, recursion=False):
+        """Verify file using gpg signature."""
+        global KEYID
+        global EMAIL
+        util.print_info("Verifying GPG signature")
+        if os.path.exists(self.package_path) is False:
+            self.print_result(False, err_msg='{} not found'.format(self.package_path))
+            return None
+        if os.path.exists(self.package_sign_path) is False and self.get_sign() is not True:
+            self.print_result(False, err_msg='{} not found'.format(self.package_sign_path))
+            return None
+        if sign_isvalid(self.package_sign_path) is False:
+            self.print_result(False, err_msg='{} is not a GPG signature'.format(self.package_sign_path))
+            try:
+                os.unlink(self.package_sign_path)
+            except Exception:
+                pass
+            return None
+        # valid signature exists at package_sign_path, operate on it now
+        keyid = get_keyid(self.package_sign_path)
+        if keyid in self.config.keyid_blocklist:
+            self.print_result(False, err_msg='KNOWNBADACTOR: {}'.format(keyid))
+            self.quit()
+
+        # default location first
+        pubkey_loc = self.pubkey_path.format(keyid)
+        cache_key = os.path.join(KEY_CACHE_DIR, pubkey_loc)
+        if os.path.exists(cache_key) and not os.path.exists(pubkey_loc):
+            shutil.copyfile(cache_key, pubkey_loc)
+        elif os.path.exists(pubkey_loc) and not os.path.exists(cache_key):
+            shutil.copyfile(pubkey_loc, cache_key)
+        elif not os.path.exists(pubkey_loc):
+            # attempt import the key interactively if set to do so
+            self.print_result(False, 'Public key {} not found'.format(keyid))
+            if not self.interactive or recursion:
+                return None
+            if attempt_key_import(keyid, pubkey_loc):
+                return self.verify(recursion=True)
+            return None
+        # public key exists or is imported, verify
+        EMAIL = get_email(pubkey_loc)
+        sign_status = verify_cli(pubkey_loc, self.package_path, self.package_sign_path)
+        if not sign_status:
+            if self.config.old_keyid:
+                compare_keys(KEYID_TRY, self.config.old_keyid)
+            self.print_result(self.package_path)
+            KEYID = KEYID_TRY
+            self.config.signature = self.key_url
+            self.config.config_opts['verify_required'] = True
+            self.config.rewrite_config_opts()
+            return True
+        else:
+            self.print_result(False, err_msg=sign_status.strerror)
+            self.quit()
+
+
+def quit_verify():
+    """Halt build due to verification being required."""
+    util.print_error("Verification required for build (verify_required option set)")
+    Verifier.quit()
+
+
+VERIFIER_TYPES = {
+    '.gz': GPGVerifier,
+    '.tgz': GPGVerifier,
+    '.tar': GPGVerifier,
+    '.bz2': GPGVerifier,
+    '.xz': GPGVerifier,
+    '.zip': GPGVerifier,
+}
+
+
+def get_file_ext(filename):
+    """Return filename extension."""
+    return os.path.splitext(filename)[1]
+
+
+def get_verifier(filename):
+    """Return verification based on filename."""
+    ext = get_file_ext(filename)
+    return VERIFIER_TYPES.get(ext, None)
+
+
+def get_input(message, default):
+    """Parse user input."""
+    try:
+        import_key = input(message)
+        if import_key == '':
+            import_key = default
+        return import_key.lower() == 'y'
+    except Exception:
+        return None
+
+
+def input_timeout(signum, frame):
+    """Handle input timeouts."""
+    print('\ninput timed out')
+    raise Exception('keyboard timed out')
+
+
+class InputGetter(object):
+    """Simple class for user input."""
+
+    def __init__(self, message='?', default='N', timeout=INPUT_GETTER_TIMEOUT):
+        """Initialize input class."""
+        self.message = message
+        self.default = default
+        self.timeout = timeout
+        signal.signal(signal.SIGALRM, input_timeout)
+
+    def get_answer(self):
+        """Read user input."""
+        signal.alarm(self.timeout)
+        inpt = get_input(self.message, self.default)
+        signal.alarm(0)
+        return inpt
+
+
+def attempt_key_import(keyid, key_fullpath):
+    """Ask user to import key."""
+    global IMPORTED
+    print(SEPT)
+    ig = InputGetter('\nDo you want to attempt to import keyid {}: (y/N) '.format(keyid))
+    import_key_answer = ig.get_answer()
+    if import_key_answer in [None, False]:
+        return False
+    with cli_gpg_ctx() as ctx:
+        err, _ = ctx.import_key(keyid)
+        if err is not None:
+            util.print_error(err.strerror)
+            return False
+        err, key_content = ctx.export_key(keyid)
+        if err is not None:
+            util.print_error(err.strerror)
+            return False
+        util.write_out(key_fullpath, key_content)
+        print('\n')
+        util.print_success('Public key id: {} was imported'.format(keyid))
+        err, content = ctx.display_keyinfo(key_fullpath)
+        if err is not None:
+            util.print_error('Unable to parse {}, will be removed'.format(key_fullpath))
+            os.unlink(key_fullpath)
+            return False
+        print("\n", content)
+        ig = InputGetter(message='\nDo you want to keep this key: (Y/n) ', default='y')
+        if ig.get_answer() is True:
+            IMPORTED = content
+            cache_key = os.path.join(KEY_CACHE_DIR, key_fullpath)
+            if not os.path.isfile(cache_key):
+                shutil.copyfile(key_fullpath, cache_key)
+            else:
+                util.print_warning(f"Re-imported key in cache: {cache_key}")
+            return True
+        else:
+            os.unlink(key_fullpath)
+    return False
+
+
+def parse_gpg_packets(filename, verbose=True):
+    """Return a list with metadata about each packet from a GPG key or signature."""
+    args = ["gpg", "--list-packets", filename]
+    try:
+        out, err = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        if err.decode('utf-8') != '' and verbose is True:
+            print(err.decode('utf-8'))
+            return None
+        out = out.decode('utf-8')
+        packets = []
+        packet = {}
+        for line in out.splitlines():
+            complete = False
+            m = re.search(r'^# off=(\d+) ctb=.* tag=.* hlen=(\d+) plen=(\d+).*$', line)
+            if m:
+                packet["offset"] = int(m.group(1))
+                packet["length"] = int(m.group(2)) + int(m.group(3))
+            m = re.search(r'^:(signature) packet:.* keyid ([0-9A-F]+)$', line)
+            if m and "type" not in packet:
+                packet["type"] = m.group(1)
+                packet["keyid"] = m.group(2)
+                complete = True
+            m = re.search(r'^:(user ID) packet: "(.*) <(.+?)>"', line)
+            if m and "type" not in packet:
+                packet["type"] = m.group(1)
+                packet["user"] = m.group(2)
+                packet["email"] = m.group(3)
+                complete = True
+            # add the packet only if we've extracted all data we need from it
+            if complete:
+                packets.append(packet)
+                packet = {}
+        return packets
+    except Exception:
+        return None
+    return None
+
+
+def get_keyid(sig_filename):
+    """Get keyid from GPG pubkey or signature file and set global KEYID_TRY."""
+    global KEYID_TRY
+    keyid = None
+    packets = parse_gpg_packets(sig_filename)
+    if packets:
+        for p in packets:
+            if "keyid" in p:
+                keyid = p["keyid"]
+                break
+    KEYID_TRY = keyid
+    return keyid.upper() if keyid else None
+
+
+def get_email(pubkey):
+    """Get user email address from GPG pubkey file and set global EMAIL."""
+    email = None
+    packets = parse_gpg_packets(pubkey)
+    if packets:
+        for p in packets:
+            if "email" in p:
+                email = p["email"]
+                break
+    return email
+
+
+def sign_isvalid(sig_filename):
+    """Get keyid from signature file."""
+    keyid = None
+    packets = parse_gpg_packets(sig_filename, verbose=False)
+    if packets:
+        keyid = packets[0].get("keyid")
+    return keyid is not None
+
+
+def filename_from_url(url):
+    """Run os.path.basename for a url."""
+    return os.path.basename(url)
+
+
+def apply_verification(verifier, **kwargs):
+    """Attempt to run verification routine."""
+    if verifier is None:
+        util.print_error("Package is not verifiable (yet)")
+    else:
+        v = verifier(**kwargs)
+        return v.verify()
+
+
+def from_disk(url, package_path, package_check, config, interactive=True):
+    """Run verification."""
+    verifier = get_verifier(package_path)
+    return apply_verification(verifier,
+                              **{
+                                  'package_path': package_path,
+                                  'package_check': package_check,
+                                  'url': url,
+                                  'interactive': interactive,
+                                  'config': config,
+                              })
+
+
+def attempt_verification_per_domain(package_path, url):
+    """Use url domain name to set verification type."""
+    netloc = urlparse(url).netloc
+    if 'download.gnome.org' in netloc:
+        domain = 'gnome.org'
+    elif 'download.qt.io' in netloc:
+        domain = 'qt.io'
+    else:
+        domain = 'unknown'
+    verifier = {
+        'gnome.org': GnomeOrgVerifier,
+        'qt.io': QtIoVerifier,
+    }.get(domain, None)
+
+    if verifier is None:
+        util.print_info('Skipping domain verification')
+        return None
+    else:
+        util.print_info('Verification based on domain {}'.format(domain))
+        return apply_verification(verifier, **{
+                                  'package_path': package_path,
+                                  'url': url})
+
+
+def get_integrity_file(package_path):
+    """Get verification filename."""
+    iter = (package_path + "." + ext for ext in ("asc", "sig", "sign", "sha256"))
+    for sign_file in iter:
+        if os.path.isfile(sign_file):
+            return sign_file
+    return None
+
+
+def check(url, config, interactive=True):
+    """Run verification based on tar file url."""
+    package_name = filename_from_url(url)
+    package_path = os.path.join(config.download_path, package_name)
+    package_check = get_integrity_file(package_path)
+    try:
+        interactive = interactive and sys.stdin.isatty()
+    except ValueError:
+        interactive = False
+    print(SEPT)
+    util.print_info('Performing package integrity verification')
+    verified = None
+    if package_check is not None:
+        verified = from_disk(url, package_path, package_check, config, interactive=interactive)
+    if not verified:
+        util.print_info('None of {}.(asc|sig|sign|sha256) is found in {}'.format(package_name, config.download_path))
+        signature_file = get_signature_file(url, config.download_path)
+        if signature_file is not None:
+            verified = from_disk(url, package_path, signature_file, config, interactive=interactive)
+            if verified is None:
+                util.print_info('Unable to find a signature')
+                verified = attempt_verification_per_domain(package_path, url)
+        else:
+            verified = attempt_verification_per_domain(package_path, url)
+
+    if verified is None and config.config_opts['verify_required']:
+        quit_verify()
+    elif verified is None:
+        print(SEPT)
+    return verified
+
+
+def parse_args():
+    """Set args for tarfile verification."""
+    parser = argparse.ArgumentParser(usage=USAGE, description=DESCRIPTION)
+    parser.add_argument('--tar', required=True,
+                        help='tar file to check signature')
+    parser.add_argument('--sig', required=True,
+                        help='Signature file')
+    parser.add_argument('--pubkey', required=False, default=None,
+                        help='Public key to use for signature verification')
+    parser.add_argument('--gnupghome', required=False, default=None,
+                        help='GNUPGHOME')
+    return parser.parse_args()
+
+
+def load_specfile(specfile):
+    """Set key and email in specfile."""
+    specfile.keyid = KEYID
+    specfile.email = EMAIL

autospec/pkg_scan.py

@@ -0,0 +1,50 @@
+#
+# pkg_scan.py - part of autospec
+# Copyright (C) 2017 Intel Corporation
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import subprocess
+
+import util
+
+
+def get_whatrequires(pkg, yum_conf):
+    """
+    Write list of packages.
+
+    Write packages that require the current package to a file
+    using dnf repoquery what-requires and --recursive commands.
+    """
+    # clean up dnf cache to avoid 'no more mirrors repo' error
+    try:
+        subprocess.check_output(['dnf', '--config', yum_conf,
+                                 '--releasever', 'clear', 'clean', 'all'])
+    except subprocess.CalledProcessError as err:
+        util.print_warning("Unable to clean dnf repo: {}, {}".format(pkg, err))
+        return
+
+    try:
+        out = subprocess.check_output(['dnf', 'repoquery',
+                                       '--config', yum_conf,
+                                       '--releasever', 'clear',
+                                       '--archlist=src', '--recursive', '--queryformat=%{NAME}',
+                                       '--whatrequires', pkg]).decode('utf-8')
+
+    except subprocess.CalledProcessError as err:
+        util.print_warning("dnf repoquery whatrequires for {} failed with: {}".format(pkg, err))
+        return
+
+    util.write_out('whatrequires', '# This file contains recursive sources that '
+                   'require this package\n' + out)

autospec/pypidata.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+import download
+import util
+
+
+def pip_env():
+    """Generate a copy of os.environ appropriate for pip."""
+    env = os.environ.copy()
+    env["PYTHON_KEYRING_BACKEND"] = "keyring.backends.null.Keyring"
+    return env
+
+
+def pkg_search(name):
+    """Query the pypi json API for name and return True if found."""
+    query = f"https://pypi.org/pypi/{name}/json/"
+    resp = download.do_curl(query)
+    if resp is not None:
+        return True
+    else:
+        return False
+
+
+def get_pypi_name(name, miss=False):
+    """Try and verify the pypi name for a given package name."""
+    # normalize the name for matching as pypi is case insensitve for search
+    name = name.lower().replace('-', '_')
+    # Common case is the name and the pypi name match
+    if pkg_search(name):
+        return name
+    # Maybe we have a prefix
+    for prefix in ["pypi_", "python_"]:
+        if name.startswith(prefix):
+            name = name[len(prefix):]
+            if pkg_search(name):
+                return name
+    # Some cases where search fails (Sphinx)
+    # Just try the name we were given
+    if miss:
+        return ""
+    return name
+
+
+def _print_command_error(cmd, proc):
+    if isinstance(cmd, list):
+        cmd = " ".join(cmd)
+    util.print_error(f"Command `{cmd}` failed:")
+    for line in proc.stderr.decode('utf-8', errors='surrogateescape').splitlines():
+        util.print_error(line)
+
+
+def get_pypi_metadata(name):
+    """Get metadata for a pypi package."""
+    show = []
+    # Create virtenv to do the pip install (needed for pip show)
+    with tempfile.TemporaryDirectory() as tdir:
+        cmd = ["virtualenv", "--no-periodic-update", tdir]
+        proc = subprocess.run(cmd, capture_output=True)
+        if proc.returncode != 0:
+            _print_command_error(cmd, proc)
+            return ""
+        cmd = f"source bin/activate && pip install {name.removeprefix('pypi_')}"
+        proc = subprocess.run(cmd, cwd=tdir, shell=True, capture_output=True,
+                              env=pip_env())
+        if proc.returncode != 0:
+            _print_command_error(cmd, proc)
+            return ""
+        cmd = f"source bin/activate &> /dev/null && pip show {name.removeprefix('pypi_')}"
+        proc = subprocess.run(cmd, cwd=tdir, shell=True, capture_output=True,
+                              env=pip_env())
+        if proc.returncode != 0:
+            _print_command_error(cmd, proc)
+            return ""
+        show = proc.stdout.decode('utf-8', errors='surrogateescape').splitlines()
+    # Parse pip show for relevent information
+    metadata = {}
+    for line in show:
+        if line.startswith("Name: "):
+            # 'Name: pypi-name'
+            # normalize names -> lowercase and dash to underscore
+            metadata["name"] = line.split()[1].lower().replace('-', '_')
+        elif line.startswith("Summary: "):
+            # 'Summary: <description of the package>'
+            try:
+                metadata["summary"] = line.split(maxsplit=1)[1]
+            except IndexError:
+                # No Summary (haven't seen this case before though)
+                metadata["summary"] = ""
+        elif line.startswith("Requires: "):
+            # 'Requires: dep1, dep2
+            # normalize names -> lowercase and dash to underscore
+            try:
+                reqs = [item.strip().lower().replace('-', '_') for item in line.split(maxsplit=1)[1].split(",")]
+            except IndexError:
+                # No Requires
+                reqs = []
+            metadata["requires"] = reqs
+
+    return json.dumps(metadata)
+
+
+def main():
+    """Standalone pypi metadata query entry point."""
+    pkg_name = sys.argv[1]
+    pypi_name = get_pypi_name(pkg_name)
+    if not pypi_name:
+        print(f"Couldn't find {pkg_name} in pypi")
+        sys.exit(1)
+    pypi_metadata = get_pypi_metadata(pypi_name)
+    print(pypi_metadata)
+
+
+if __name__ == '__main__':
+    main()

autospec/qt_modules

@@ -0,0 +1,68 @@
+# This file maps the qmake entry for the QT or QT_PRIVATE variable to the
+# pkgconfig equivalent, to be added to BuildRequires. Please keep sorted.
+# This file is sorted with LC_COLLATE=C
+# Format: <qmake key>, <pkgconfig value>
+3danimation, Qt63DAnimation
+3dcore, Qt63DCore
+3dextras, Qt63DExtras
+3dinput, Qt63DInput
+3dlogic, Qt63DLogic
+3dquick, Qt63DQuick
+3dquickanimation, Qt63DQuickAnimation
+3dquickextras, Qt63DQuickExtras
+3dquickinput, Qt63DQuickInput
+3dquickrender, Qt63DQuickRender
+3dquickscene2d, Qt63DQuickScene2D
+3drender, Qt63DRender
+bluetooth, Qt6Bluetooth
+charts, Qt6Charts
+concurrent, Qt6Concurrent
+core, Qt6Core
+datavisualization, Qt6DataVisualization
+dbus, Qt6DBus
+declarative, Qt6Declarative
+designer, Qt6Designer
+gamepad, Qt6Gamepad
+gui, Qt6Gui
+help, Qt6Help
+location, Qt6Location
+multimedia, Qt6Multimedia
+multimediawidgets, Qt6MultimediaWidgets
+network, Qt6Network
+networkauth, Qt6NetworkAuth
+nfc, Qt6Nfc
+opengl, Qt6OpenGL
+openglextensions, Qt6OpenGLExtensions
+positioning, Qt6Positioning
+printsupport, Qt6PrintSupport
+purchasing, Qt6Purchasing
+qml, Qt6Qml
+qmltest, Qt6QuickTest
+quick, Qt6Quick
+quickcontrols2, Qt6QuickControls2
+quickwidgets, Qt6QuickWidgets
+remoteobjects, Qt6RemoteObjects
+script, Qt6Script
+scripttools, Qt6ScriptTools
+scxml, Qt6Scxml
+sensors, Qt6Sensors
+serialbus, Qt6SerialBus
+serialport, Qt6SerialPort
+sql, Qt6Sql
+svg, Qt6Svg
+testlib, Qt6Test
+texttospeech, Qt6TextToSpeech
+uitools, Qt6UiTools
+waylandclient, Qt6WaylandClient
+waylandcompositor, Qt6WaylandCompositor
+webchannel, Qt6WebChannel
+webengine, Qt6WebEngine
+webenginecore, Qt6WebEngineCore
+webenginewidgets, Qt6WebEngineWidgets
+webkit, Qt6WebKit
+webkitwidgets, Qt6WebKitWidgets
+websockets, Qt6WebSockets
+widgets, Qt6Widgets
+x11extras, Qt6X11Extras
+xml, Qt6Xml
+xmlpatterns, Qt6XmlPatterns

autospec/specdescription.py

@@ -27,240 +27,242 @@
 #
 
 import os
+import re
+
 import license
+import util
 
-default_group = "Development/Tools"
 default_description = "No detailed description available"
 default_description_score = 0
 default_summary = "No detailed summary available"
 default_summary_score = 0
 
 
-def clean_license_string(str):
+def clean_license_string(lic):
-
+    """Clean up license string by replacing substrings."""
-    if str.find("same as") >= 0:
+    if lic.find("same as") >= 0:
         return ""
-    str = str.replace(" (", "(")
-    str = str.replace(" v2", "-2")
-    str = str.replace(" v3", "-3")
-    str = str.replace(" 2", "-2")
-    str = str.replace(" 3", "-3")
-    str = str.replace(" <", "<")
-    str = str.replace(" >", ">")
-    str = str.replace("= ", "=")
-    str = str.replace("GPL(>=-2)", "GPL-2.0+")
 
-    str = str.replace("Modified", "")
+    reps = [(" (", "("),
-    str = str.replace("OSI", "")
+            (" v2", "-2"),
-    str = str.replace("Approved", "")
+            (" v3", "-3"),
-    str = str.replace("Simplified", "")
+            (" 2", "-2"),
-    str = str.replace("file", "")
+            (" 3", "-3"),
-    str = str.replace("LICENSE", "")
+            (" <", "<"),
+            (" >", ">"),
+            ("= ", "="),
+            ("GPL(>=-2)", "GPL-2.0+"),
+            ("Modified", ""),
+            ("OSI", ""),
+            ("Approved", ""),
+            ("Simplified", ""),
+            ("file", ""),
+            ("LICENSE", "")]
 
-    return str
+    for sub, rep in reps:
+        lic = lic.replace(sub, rep)
+
+    return lic
 
 
-#
+def assign_summary(summary, score):
-# Parse any existing RPM specfiles
+    """Assign summary to default_summary if score is greater than default_summary_score."""
-#
-
-def description_from_spec(specfile):
-    global default_description
     global default_summary
     global default_summary_score
+    if score > default_summary_score:
+        default_summary = summary
+        default_summary_score = score
+
+
+def assign_description(description, score):
+    """Assign description to default_description if score is greater than default_description_score."""
+    global default_description
     global default_description_score
+    if score > default_description_score:
+        default_description = description
+        default_description_score = score
+
+
+def description_from_spec(specfile, translations, blacklist):
+    """Parse any existing RPM specfiles."""
+    try:
+        with util.open_auto(specfile, 'r') as specfd:
+            lines = specfd.readlines()
+    except FileNotFoundError:
+        return
+
     specdesc = ""
-    phase = 0
+    section = False
-    file = open(specfile, "r", encoding="latin-1")
+    for line in lines:
-    for line in file.readlines():
         if line.startswith("#"):
             continue
 
         if line.startswith("%"):
-            phase = 0
+            section = False
 
-        if line.startswith("License:") and line.find("Copyright") < 0 and line.find("see ") < 0 and line.find("(") < 0:
+        excludes = ["Copyright", "see ", "("]
+        if line.startswith("License:") and not any(e in line for e in excludes):
             splits = line.split(":")[1:]
             words = ":".join(splits).strip()
-            if words in license.license_translations:
+            if words in translations:
-                print("Adding license from spec:", words)
+                if license.add_license(words, translations, blacklist):
-                license.add_license(words)
+                    print("Added license from spec:", words)
             else:
                 words = clean_license_string(words).split()
                 for word in words:
-                    if word.find(":") < 0 or word.startswith('http'):
+                    if ":" not in word and not word.startswith("@"):
-                        print("Adding license from spec:", word)
+                        if license.add_license(word, translations, blacklist):
-                        license.add_license(word)
+                            print("Added license from spec:", word)
 
-        if line.startswith("Summary: ") and default_summary_score < 4:
+        if line.startswith("Summary: "):
-            default_summary = line[9:]
+            assign_summary(line[9:], 4)
-            default_summary_score = 4
-
-        if phase == 1:
-            specdesc = specdesc + line
 
+        specdesc += line if section else ""
+        # Check for %description after assigning the line to specdesc so the
+        # %description string is not included
         if line.endswith("%description\n"):
-            phase = 1
+            section = True
-    if default_description_score < 4:
+
-        default_description = specdesc
+    if len(specdesc) > 10:
-        default_description_score = 4
+        assign_description(specdesc, 4)
-    file.close()
 
 
-def description_from_pkginfo(specfile):
+def description_from_pkginfo(pkginfo, translations, blacklist):
-    global default_description
+    """Parse existing package info files."""
-    global default_summary
+    try:
-    global default_summary_score
+        with util.open_auto(pkginfo, 'r') as pkgfd:
-    global default_description_score
+            lines = pkgfd.readlines()
-    specdesc = ""
+    except FileNotFoundError:
-    phase = 0
+        return
-    file = open(specfile, "r", encoding="latin-1")
-    for line in file.readlines():
-        if line.find(":") and phase == 1:
-            phase = 0
 
-        if line.lower().startswith("license:") and line.find("Copyright") < 0 and line.find("see ") < 0:
+    pkginfo = ""
+    section = False
+    for line in lines:
+        if ":" in line and section:
+            section = False
+
+        excludes = ["Copyright", "see "]
+        if line.lower().startswith("license:") and not any(e in line for e in excludes):
             splits = line.split(":")[1:]
             words = ":".join(splits).strip()
-            if words in license.license_translations:
+            if words in translations:
-                print("Adding license from PKG-INFO:", words)
+                if license.add_license(words, translations, blacklist):
-                license.add_license(words)
+                    print("Added license from PKG-INFO:", words)
             else:
                 words = clean_license_string(words).split()
                 for word in words:
-                    if word.find(":") < 0:
+                    if ":" not in word:
-                        print("Adding license from PKG-INFO:", word)
+                        if license.add_license(word, translations, blacklist):
-                        license.add_license(word)
+                            print("Added license from PKG-INFO:", word)
 
-        if line.startswith("Summary: ") and default_summary_score < 4:
+        for sub in ["Summary: ", "abstract: "]:
-            default_summary = line[9:]
+            if line.startswith(sub):
-            default_summary_score = 4
+                assign_summary(line[len(sub):].strip(), 4)
-
-        if line.startswith("abstract:") and default_summary_score < 4:
-            default_summary = line[9:].strip()
-            default_summary_score = 4
-
-        if phase == 1:
-            specdesc = specdesc + line
 
+        pkginfo += line if section else ""
         if line.startswith("Description:"):
-            phase = 1
+            section = True
-    if default_description_score < 4 and len(specdesc) > 10:
-        default_description = specdesc
-        default_description_score = 4
-    file.close()
 
-#
+    if len(pkginfo) > 10:
-# Parse pkgconfig files for Description: lines
+        assign_description(pkginfo, 4)
-#
 
 
 def summary_from_pkgconfig(pkgfile, package):
-    global default_summary
+    """Parse pkgconfig files for Description: lines."""
-    global default_summary_score
+    try:
-    score = 2
+        with util.open_auto(pkgfile, "r") as pkgfd:
+            lines = pkgfd.readlines()
+    except FileNotFoundError:
+        return
 
-    if pkgfile.find(package + ".pc") >= 0:
+    score = 3 if package + ".pc" in pkgfile else 2
-        score = 3
+    for line in lines:
-
+        if line.startswith("Description:"):
-    file = open(pkgfile, "r")
+            assign_summary(line[13:], score)
-    for line in file.readlines():
+            # Score will not increase, stop trying
-        if line.startswith("Description:") and default_summary_score < score:
+            break
-            default_summary = line[13:]
-            default_summary_score = score
-    file.close()
 
 
-def summary_from_R(pkgfile, package):
+def summary_from_R(pkgfile):
-    global default_summary
+    """Parse DESCRIPTION file for Title: lines."""
-    global default_summary_score
+    try:
-    score = 2
+        with util.open_auto(pkgfile, "r") as pkgfd:
+            lines = pkgfd.readlines()
+    except FileNotFoundError:
+        return
 
-    if pkgfile.find("DESCRIPTION") >= 0:
+    for line in lines:
-        score = 3
+        if line.startswith("Title:"):
-
+            assign_summary(line[7:], 3)
-    file = open(pkgfile, "r", encoding="latin-1")
+            # Score will not increase, stop trying
-    for line in file.readlines():
+            break
-        if line.startswith("Title:") and default_summary_score < score:
-            default_summary = line[7:]
-            default_summary_score = score
-    file.close()
 
 
-#
-# some lines from a readme are just boilerplate and should be skipped
-#
 def skipline(line):
-    if line.find("Copyright") >= 0:
+    """Skip boilerplate readme lines."""
-        return 1
-    if line.find("Free Software Foundation, Inc.") >= 0:
-        return 1
-    if line.find("Copying and distribution of") >= 0:
-        return 1
-    if line.find("are permitted in any") >= 0:
-        return 1
-    if line.find("notice and this notice") >= 0:
-        return 1
-    if line.find("README") >= 0:
-        return 1
-    if line.find("-*-") >= 0:
-        return 1
-
     if line.endswith("introduction"):
-        return 1
+        return True
-    return 0
+
+    skips = ["Copyright",
+             "Free Software Foundation, Inc.",
+             "Copying and distribution of",
+             "are permitted in any",
+             "notice and this notice",
+             "README",
+             "-*-"]
+    return any(s in line for s in skips)
 
 
-#
-# Try to pick the first paragraph or two from the readme file
-#
 def description_from_readme(readmefile):
-    global default_description
+    """Try to pick the first paragraph or two from the readme file."""
-    global default_description_score
+    try:
-    state = 0
+        with util.open_auto(readmefile, "r") as readmefd:
+            lines = readmefd.readlines()
+    except FileNotFoundError:
+        return
+
+    section = False
     desc = ""
-    score = 1
+    for line in lines:
-
+        if section and len(line) < 2 and len(desc) > 80:
-    if readmefile.lower().endswith("readme"):
+            # If we are in a section and encounter a new line, break as long as
-        score = 1.5
+            # we already have a description > 80 characters.
-
+            break
-    file = open(readmefile, "r", encoding="latin-1")
+        if not section and len(line) > 2:
-    for line in file.readlines():
+            # Found the first paragraph hopefully
-        if state == 1 and len(line) < 2 and len(desc) > 80:
+            section = True
-            state = 2
+        if section:
-        if state == 0 and len(line) > 2:
+            # Copy all non-empty lines into the description
-            state = 1
-        if state == 1:
             if skipline(line) == 0 and len(line) > 2:
                 desc = desc + line.strip() + "\n"
 
-    if default_description_score < score:
+    score = 1.5 if readmefile.lower().endswith("readme") else 1
-        default_description = desc
+    assign_description(desc, score)
-        default_description_score = score
-    file.close()
-
-#
-# Scan the project directory for things we can use to guess a description
-# and summary
-#
 
 
-def scan_for_description(package, dir):
+def scan_for_description(package, dirn, translations, blacklist):
-    global default_summary
+    """Scan the project directory for things we can use to guess a description and summary."""
-    for dirpath, dirnames, files in os.walk(dir):
+    test_pat = re.compile(r"tests?")
+    dirpath_seen = ""
+    for dirpath, dirnames, files in os.walk(dirn):
+        if dirpath_seen != dirpath:
+            dirpath_seen = dirpath
+            dirnames[:] = [d for d in dirnames if not re.match(test_pat, d)]
         for name in files:
+            if name.lower().endswith(".pdf"):
+                continue
             if name.lower().endswith(".spec"):
-                description_from_spec(os.path.join(dirpath, name))
+                description_from_spec(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith("pkg-info"):
-                description_from_pkginfo(os.path.join(dirpath, name))
+                description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith("meta.yml"):
-                description_from_pkginfo(os.path.join(dirpath, name))
+                description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith("description"):
-                description_from_pkginfo(os.path.join(dirpath, name))
+                description_from_pkginfo(os.path.join(dirpath, name), translations, blacklist)
             if name.lower().endswith(".pc"):
                 summary_from_pkgconfig(os.path.join(dirpath, name), package)
             if name.startswith("DESCRIPTION"):
-                summary_from_R(os.path.join(dirpath, name), package)
+                summary_from_R(os.path.join(dirpath, name))
             if name.lower().endswith(".pc.in"):
                 summary_from_pkgconfig(os.path.join(dirpath, name), package)
             if name.lower().startswith("readme"):
@@ -268,22 +270,14 @@ def scan_for_description(package, dir):
 
     print("Summary     :", default_summary.strip())
 
-#  print("Summary: ", default_summary)
-#  print("%description")
-#  print(default_description)
 
-
+def load_specfile(specfile, description, summary):
-def write_summary(file):
+    """Load specfile with parse results."""
-    global default_summary
+    if description:
-    global default_group
+        specfile.default_desc = "\n".join(description)
-    file.write("Summary  : " + default_summary.strip() + "\n")
-    file.write("Group    : " + default_group.strip() + "\n")
-
-
-def write_description(file, package=""):
-    global default_description
-    if len(package) == 0:
-        file.write("\n%description\n" + default_description.strip() + "\n")
     else:
-        file.write("\n%description " + package +
+        specfile.default_desc = default_description
-                   "\n Subpackage " + package + "\n\n")
+    if summary:
+        specfile.default_sum = summary[0]
+    else:
+        specfile.default_sum = default_summary

autospec/tarball.py

@@ -17,281 +17,437 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import build
+import bz2
-import buildpattern
+import configparser
-import buildreq
-import files
-import glob
-import hashlib
 import os
 import re
-import shutil
+import sys
-import subprocess
+import tarfile
-import urllib
+import zipfile
-import urllib.request
-from util import call
 
-name = ""
+import download
-rawname = ""
+from util import call, do_regex, get_sha1sum, print_fatal, write_out
-version = ""
-release = "1"
-url = ""
-path = ""
-tarball_prefix = ""
-gcov_file = ""
 
 
-def get_sha1sum(filename):
+class Source():
-    sh = hashlib.sha1()
+    """Holds data and methods for source code or archives management."""
-    with open(filename, "rb") as f:
+
-        sh.update(f.read())
+    def __init__(self, url, destination, path, pattern=None):
-    return sh.hexdigest()
+        """Set default values for source file."""
+        self.url = url
+        self.destination = destination
+        self.path = path
+        self.pattern = pattern
+        self.type = None
+        self.prefix = None
+        self.subdir = None
+
+        # Extra  compressed archives
+        if not self.destination.startswith(':'):
+            self.set_type()
+            self.set_prefix()
+
+    def set_type(self):
+        """Determine compression type."""
+        if self.url.lower().endswith(('.zip', 'jar')):
+            self.type = 'zip'
+        elif self.url.lower().endswith(('.bz2')) and not self.url.lower().endswith(('.tar.bz2')):
+            self.type = 'bz2'
+        else:
+            self.type = 'tar'
+
+    def set_prefix(self):
+        """Determine the prefix and subdir if no prefix."""
+        prefix_method = getattr(self, 'set_{}_prefix'.format(self.type))
+        prefix_method()
+        # When there is no prefix, create subdir
+        if not self.prefix:
+            self.subdir = os.path.splitext(os.path.basename(self.path))[0]
+
+    def set_tar_prefix(self):
+        """Determine prefix folder name of tar file."""
+        if tarfile.is_tarfile(self.path):
+            with tarfile.open(self.path, 'r') as content:
+                lines = content.getnames()
+                # When tarball is not empty
+                if len(lines) == 0:
+                    print_fatal("Tar file doesn't appear to have any content")
+                    sys.exit(1)
+                elif len(lines) > 1:
+                    if 'package.xml' in lines and self.pattern in ['phpize']:
+                        lines.remove('package.xml')
+                    self.prefix = os.path.commonpath(lines)
+        else:
+            print_fatal("Not a valid tar file.")
+            sys.exit(1)
+
+    def set_bz2_prefix(self):
+        """No prefix for plain bz2 archives."""
+
+    def set_zip_prefix(self):
+        """Determine prefix folder name of zip file."""
+        if zipfile.is_zipfile(self.path):
+            with zipfile.ZipFile(self.path, 'r') as content:
+                lines = content.namelist()
+                # When zipfile is not empty
+                if len(lines) > 0:
+                    self.prefix = os.path.commonpath(lines)
+                else:
+                    print_fatal("Zip file doesn't appear to have any content")
+                    sys.exit(1)
+        else:
+            print_fatal("Not a valid zip file.")
+            sys.exit(1)
+
+    def extract(self, base_path):
+        """Prepare extraction path and call specific extraction method."""
+        if not self.prefix:
+            extraction_path = os.path.join(base_path, self.subdir)
+        else:
+            extraction_path = base_path
+
+        extract_method = getattr(self, 'extract_{}'.format(self.type))
+        extract_method(extraction_path)
+
+    def extract_tar(self, extraction_path):
+        """Extract tar in path."""
+        with tarfile.open(self.path) as content:
+            content.extractall(path=extraction_path)
+
+    def extract_bz2(self, extraction_path):
+        """Extract plain bz2 file in path."""
+        with bz2.BZ2File(self.path, 'rb') as content:
+            data = content.read()
+            newfile = self.path.rsplit('/', 1)[1]
+            newfile = newfile.rsplit('.bz2', 1)[0]
+            with open(os.path.join(extraction_path), mode='wb') as f:
+                f.write(data)
+
+    def extract_zip(self, extraction_path):
+        """Extract zip in path."""
+        with zipfile.ZipFile(self.path, 'r') as content:
+            content.extractall(path=extraction_path)
 
 
-def really_download(url, destination, tarfile):
+def convert_version(ver_str, name):
-    with urllib.request.urlopen(url) as response:
+    """Remove disallowed characters from the version."""
-        with open(destination, "wb") as dfile:
+    # banned substrings. It is better to remove these here instead of filtering
-            dfile.write(response.read())
+    # them out with expensive regular expressions
+    banned_subs = ["x86.64", "source", "src", "all", "bin", "release", "rh",
+                   "ga", ".ce", "lcms", "onig", "linux", "gc", "sdk", "orig",
+                   "jurko", "%2f", "%2F", "%20", "x265"]
+
+    # package names may be modified in the version string by adding "lib" for
+    # example. Remove these from the name before trying to remove the name from
+    # the version
+    name_mods = ["lib", "core", "pom", "opa-"]
+
+    # enforce lower-case strings to make them easier to standardize
+    ver_str = ver_str.lower()
+    # remove the package name from the version string
+    ver_str = ver_str.replace(name.lower(), '')
+    # handle modified name substrings in the version string
+    for mod in name_mods:
+        ver_str = ver_str.replace(name.replace(mod, ""), "")
+
+    # replace illegal characters
+    ver_str = ver_str.strip().replace('-', '.').replace('_', '.')
+
+    # remove banned substrings
+    for sub in banned_subs:
+        ver_str = ver_str.replace(sub, "")
+
+    # remove consecutive '.' characters
+    while ".." in ver_str:
+        ver_str = ver_str.replace("..", ".")
+
+    return ver_str.strip(".")
 
 
-def check_or_get_file(url, tarfile):
+class Content():
-    tarball_path = build.download_path + "/" + tarfile
+    """Detect static information about the project."""
-    if not os.path.isfile(tarball_path):
-        really_download(url, tarball_path, tarfile)
-    return tarball_path
 
+    def __init__(self, url, name, version, archives, config, base_path):
+        """Initialize Default content settings."""
+        self.name = name
+        self.rawname = ""
+        self.version = version
+        self.release = "1"
+        self.url = url
+        self.path = ""
+        self.tarball_prefix = ""
+        self.gcov_file = ""
+        self.archives = archives
+        self.giturl = ""
+        self.repo = ""
+        self.domain = ""
+        self.prefixes = dict()
+        self.config = config
+        self.base_path = base_path
+        self.autogenerated_tarball = None
 
-def build_untar(tarball_path):
+    def write_upstream(self, sha, tarfile, mode="w"):
-    tarball_contents = subprocess.check_output(
+        """Write the upstream hash to the upstream file."""
-        ["tar", "-tf", tarball_path], universal_newlines=True)
+        write_out(os.path.join(self.config.download_path, "upstream"),
-    extract_cmd = "tar --directory={0} -xf {1}".format(build.base_path, tarball_path)
+                  os.path.join(sha, tarfile) + "\n", mode=mode)
-    if tarball_contents:
-        tarball_prefix = tarball_contents.splitlines()[0].rsplit("/")[0]
-        if tarball_prefix == ".":
-            tarball_prefix = tarball_contents.splitlines()[0].rsplit("/")[1]
-    return extract_cmd, tarball_prefix
 
+    def extract_sources(self, main_src, archives_src):
+        """Extract sources."""
+        full_list_src = [main_src] + archives_src
+        for src in full_list_src:
+            if src.destination != ':':
+                src.extract(self.base_path)
 
-def download_tarball(url_argument, name_argument, archives, target_dir):
+    def check_or_get_file(self, upstream_url, tarfile, mode="w"):
-    global name
+        """Download tarball from url unless it is present locally."""
-    global rawname
+        tarball_path = self.config.download_path + "/" + tarfile
-    global version
+        if not os.path.isfile(tarball_path):
-    global url
+            download.do_curl(upstream_url, dest=tarball_path, is_fatal=True)
-    global path
+            self.write_upstream(get_sha1sum(tarball_path), tarfile, mode)
-    global tarball_prefix
+        else:
-    global gcov_file
+            self.write_upstream(get_sha1sum(tarball_path), tarfile, mode)
+        return tarball_path
 
-    url = url_argument
+    def process_main_source(self, url):
-    tarfile = os.path.basename(url)
+        """Download and get important information from main source code."""
-    pattern_options = [
+        src_path = self.check_or_get_file(url, os.path.basename(url))
-        r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)",
+        main_src = Source(url, '', src_path, self.config.default_pattern)
-        r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)",
+        return main_src
-        r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar",
-        r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar",
-    ]
-    for pattern in pattern_options:
-        p = re.compile(pattern)
-        m = p.search(tarfile)
-        if m:
-            name = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
-            break
 
-    rawname = name
+    def process_autogenerated_source(self, url):
-    # R package
+        """Download any autogenerated source tarball for comparison."""
-    if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0:
+        autogenerated_src = None
-        buildpattern.set_build_pattern("R", 10)
+        if url:
-        files.want_dev_split = 0
+            src_path = self.check_or_get_file(url, os.path.basename(url), mode="")
-        buildreq.add_buildreq("clr-R-helpers")
+            autogenerated_src = Source(url, '../autogenerated-tmp', src_path, self.config.default_pattern)
-        p = re.compile(r"([A-Za-z0-9]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz")
+        return autogenerated_src
-        m = p.search(tarfile)
-        if m:
-            name = "R-" + m.group(1).strip()
-            rawname = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
 
-    if url_argument.find("pypi.python.org") > 0:
+    def print_header(self):
-        buildpattern.set_build_pattern("distutils", 10)
+        """Print header for autospec run."""
+        print("\n")
+        print("Processing", self.url)
+        print("=" * 105)
+        print("Name        :", self.name)
+        print("Version     :", self.version)
+        print("Prefix      :", self.tarball_prefix)
 
-    if url_argument.find(".cpan.org/CPAN/") > 0:
+    def set_giturl_and_domain(self):
-        buildpattern.set_build_pattern("cpan", 10)
+        """Set giturl and domain from the config (needs config refactor)."""
-        if name:
+        options_path = os.path.join(self.config.download_path, 'options.conf')
-            name = "perl-" + name
+        if os.path.exists(options_path):
-    if url_argument.find(".metacpan.org/") > 0:
+            config_f = configparser.ConfigParser(interpolation=None)
-        buildpattern.set_build_pattern("cpan", 10)
+            config_f.read(options_path)
-        if name:
+            if "package" in config_f.sections():
+                if "giturl" in config_f["package"]:
+                    self.giturl = config_f["package"].get("giturl")
+                if "domain" in config_f["package"]:
+                    self.domain = config_f["package"].get("domain")
+
+    def name_and_version(self, filemanager):
+        """Parse the url for the package name and version."""
+        tarfile = os.path.basename(self.url)
+
+        # If both name and version overrides are set via commandline, set the name
+        # and version variables to the overrides and bail. If only one override is
+        # set, continue to auto detect both name and version since the URL parsing
+        # handles both. In this case, wait until the end to perform the override of
+        # the one that was set. An extra conditional, that version_arg is a string
+        # is added to enable a package to have multiple versions at the same time
+        # for some language ecosystems.
+        if self.name and self.version:
+            # rawname == name in this case
+            self.rawname = self.name
+            self.version = convert_version(self.version, self.name)
+            return
+
+        name = self.name
+        self.rawname = self.name
+        version = ""
+        # it is important for the more specific patterns to come first
+        pattern_options = [
+            # handle font packages with names ending in -nnndpi
+            r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
+            r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
+        ]
+        match = do_regex(pattern_options, tarfile)
+        if match:
+            name = match.group(1).strip()
+            version = convert_version(match.group(2), name)
+
+        # R package
+        if "cran.r-project.org" in self.url or "cran.rstudio.com" in self.url and name:
+            filemanager.want_dev_split = False
+            self.rawname = name
+            name = "R-" + name
+
+        if ".cpan.org/" in self.url or ".metacpan.org/" in self.url and name:
             name = "perl-" + name
 
-    if url_argument.find("github.com") > 0:
+        if "github.com" in self.url:
-        p = re.compile(r"https://github.com/.*/(.*?)/archive/v?(.*).tar")
+            # define regex accepted for valid packages, important for specific
-        m = p.search(url_argument)
+            # patterns to come before general ones
-        if m:
+            github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/refs/tags/[vVrR]?(.*)\.tar",
-            name = m.group(1).strip()
+                               r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
-            version = m.group(2).strip()
+                               r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z_]*-(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
+                               r"https?://github.com/(.*)/.*-downloads/releases/download/.*?/(.*)-(.*).tar",
+                               r"https?://github.com/(.*)/(.*?)/releases/download/(.*)/",
+                               r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar"]
 
-    if url_argument.find("bitbucket.org") > 0:
+            match = do_regex(github_patterns, self.url)
-        p = re.compile(r"https://bitbucket.org/.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9_.]*).tar")
+            if match:
-        m = p.search(url_argument)
+                self.repo = match.group(2).strip()
-        if m:
+                if self.repo not in name:
-            name = m.group(1).strip()
+                    # Only take the repo name as the package name if it's more descriptive
-            version = m.group(2).strip().replace('_', '.')
+                    name = self.repo
+                elif name != self.repo:
+                    name = re.sub(r"release-", '', name)
+                    name = re.sub(r"\d*$", '', name)
+                self.rawname = name
+                # Identify the auto-generated tarball URL for comparison
+                if "/releases/download/" in self.url:
+                    self.autogenerated_tarball = "https://github.com/" + match.group(1).strip() + "/" + self.repo + "/archive/refs/tags/" + match.group(3).strip() + ".tar.gz"
+                version = match.group(3).replace(name, '')
+                if "/archive/" not in self.url:
+                    version = re.sub(r"^[-_.a-zA-Z]+", "", version)
+                version = convert_version(version, name)
+                if not self.giturl:
+                    self.giturl = "https://github.com/" + match.group(1).strip() + "/" + self.repo + ".git"
 
-    # ruby
+        # SQLite tarballs use 7 digit versions, e.g 3290000 = 3.29.0, 3081002 = 3.8.10.2
-    if url_argument.find("rubygems.org/") > 0:
+        if "sqlite.org" in self.url:
-        buildpattern.set_build_pattern("ruby", 10)
+            major = version[0]
-        p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem")
+            minor = version[1:3].lstrip("0").zfill(1)
-        m = p.search(tarfile)
+            patch = version[3:5].lstrip("0").zfill(1)
-        if m:
+            build = version[5:7].lstrip("0")
-            buildreq.add_buildreq("ruby")
+            version = major + "." + minor + "." + patch + "." + build
-            buildreq.add_buildreq("rubygem-rdoc")
+            version = version.strip(".")
-            name = "rubygem-" + m.group(1).strip()
-            rawname = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
 
-    # override from commandline
+        # Construct gitlab giturl for GNOME projects, and update previous giturls
-    if name_argument and name_argument[0] != name:
+        # that pointed to the GitHub mirror.
-        pattern = name_argument[0] + r"[\-]*(.*)\.(tgz|tar|zip)"
+        if "download.gnome.org" in self.url:
-        p = re.compile(pattern)
+            if not self.giturl or "github.com/GNOME" in self.giturl or "git.gnome.org" in self.giturl:
-        m = p.search(tarfile)
+                self.giturl = "https://gitlab.gnome.org/GNOME/{}".format(name)
-        if m:
-            name = name_argument[0]
-            rawname = name
-            version = m.group(1).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
-            if version.startswith('.'):
-                version = version[1:]
-        else:
-            name = name_argument[0]
 
-    if not name:
+        if "mirrors.kernel.org" in self.url:
-        split = url_argument.split('/')
+            m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", self.url)
-        if len(split) > 3 and split[-2] in ('archive', 'tarball'):
+            if m:
-            name = split[-3]
+                name = m.group(1).strip()
-            version = split[-1]
+                version = convert_version(m.group(2), name)
-            if version.startswith('v'):
-                version = version[1:]
-            # remove extension
-            version = '.'.join(version.split('.')[:-1])
-            if version.endswith('.tar'):
-                version = '.'.join(version.split('.')[:-1])
 
-    b = version.find("-")
+        if "sourceforge.net" in self.url:
-    if b >= 0:
+            scf_pats = [r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz",
-        b = b + 1
+                        r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz"]
-        version = version[b:]
+            match = do_regex(scf_pats, self.url)
+            if match:
+                name = match.group(1).strip()
+                version = convert_version(match.group(2), name)
 
-    if version[0] in ['v', 'r']:
+        if "bitbucket.org" in self.url:
-        version = version[1:]
+            bitbucket_pats = [r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)",
+                              r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)"]
 
-    assert name != ""
+            match = do_regex(bitbucket_pats, self.url)
+            if match:
+                name = match.group(1).strip()
+                version = convert_version(match.group(2), name)
 
-    if not target_dir:
+        if "gitlab.com" in self.url:
-        build.download_path = os.getcwd() + "/" + name
+            # https://gitlab.com/leanlabsio/kanban/-/archive/1.7.1/kanban-1.7.1.tar.gz
-    else:
+            m = re.search(r"gitlab\.com/.*/(.*)/-/archive/(?:VERSION_|[vVrR])?(.*)/", self.url)
-        build.download_path = target_dir
+            if m:
-    call("mkdir -p %s" % build.download_path)
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    gcov_path = build.download_path + "/" + name + ".gcov"
+        if "git.sr.ht" in self.url:
-    if os.path.isfile(gcov_path):
+            # https://git.sr.ht/~sircmpwn/scdoc/archive/1.9.4.tar.gz
-        gcov_file = name + ".gcov"
+            m = re.search(r"git\.sr\.ht/.*/(.*)/archive/(.*).tar.gz", self.url)
+            if m:
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    tarball_path = check_or_get_file(url, tarfile)
+        if "pigeonhole.dovecot.org" in self.url:
-    sha1 = get_sha1sum(tarball_path)
+            # https://pigeonhole.dovecot.org/releases/2.3/dovecot-2.3-pigeonhole-0.5.20.tar.gz
-    with open(build.download_path + "/upstream", "w") as file:
+            if m := re.search(r"pigeonhole\.dovecot\.org/releases/.*/dovecot-[\d\.]+-(\w+)-([\d\.]+)\.[^\d]", self.url):
-        file.write(sha1 + "/" + tarfile + "\n")
+                name = m.group(1).strip()
+                version = convert_version(m.group(2), name)
 
-    tarball_prefix = name + "-" + version
+        if ".ezix.org" in self.url:
-    if tarfile.lower().endswith('.zip'):
+            # https://www.ezix.org/software/files/lshw-B.02.19.2.tar.gz
-        tarball_contents = subprocess.check_output(
+            if m := re.search(r"(\w+)-[A-Z]\.(\d+(?:\.\d+)+)", self.url):
-            ["unzip", "-l", tarball_path], universal_newlines=True)
+                name = m.group(1).strip()
-        if tarball_contents and len(tarball_contents.splitlines()) > 3:
+                version = convert_version(m.group(2), name)
-            tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1]
-        extract_cmd = "unzip -d {0} {1}".format(build.base_path, tarball_path)
 
-    elif tarfile.lower().endswith('.gem'):
+        if self.name and not version:
-        tarball_contents = subprocess.check_output(
+            # In cases where we have a name but no version
-            ["gem", "unpack", "--verbose", tarball_path], universal_newlines=True)
+            # use what is after the name.
-        extract_cmd = "gem unpack --target={0} {1}".format(build.base_path, tarball_path)
+            # https://invisible-mirror.net/archives/lynx/tarballs/lynx2.8.9rel.1.tar.gz
-        if tarball_contents:
+            postname = tarfile.split(self.name)[-1]
-            tarball_prefix = tarball_contents.splitlines()[-1].rsplit("/")[-1]
+            no_extension = os.path.splitext(postname)[0]
-            if tarball_prefix.endswith("'"):
+            if no_extension.endswith('.tar'):
-                tarball_prefix = tarball_prefix[:-1]
+                no_extension = os.path.splitext(no_extension)[0]
-    else:
+            version = convert_version(no_extension, self.name)
-        extract_cmd, tarball_prefix = build_untar(tarball_path)
 
-    print("\n")
+        # override name and version from commandline
+        self.name = self.name if self.name else name
+        self.version = self.version if self.version else version
 
-    print("Processing", url_argument)
+    def set_gcov(self):
-    print(
+        """Set the gcov file name."""
-        "=============================================================================================")
+        gcov_path = os.path.join(self.config.download_path, self.name + ".gcov")
-    print("Name        :", name)
+        if os.path.isfile(gcov_path):
-    print("Version     :", version)
+            self.gcov_file = self.name + ".gcov"
-    print("Prefix      :", tarball_prefix)
 
-    with open(build.download_path + "/Makefile", "w") as file:
+    def process_archives(self):
-        file.write("PKG_NAME := " + name + "\n")
+        """Process extra sources needed by package."""
-        file.write("URL := " + url_argument + "\n")
+        src_objects = []
-        file.write("ARCHIVES :=")
-        for archive in archives:
-            file.write(" {}".format(archive))
-        file.write("\n")
-        file.write("\n")
-        file.write("include ../common/Makefile.common\n")
 
-    shutil.rmtree("{}".format(build.base_path), ignore_errors=True)
+        full_archives = self.archives
-    os.makedirs("{}".format(build.output_path))
+        # Download and extract full list
-    call("mkdir -p %s" % build.download_path)
+        for arch_url, destination in zip(full_archives[::2], full_archives[1::2]):
-    call(extract_cmd)
+            src_path = self.check_or_get_file(arch_url, os.path.basename(arch_url), mode="a")
+            # Create source object and extract archive
+            archive = Source(arch_url, destination, src_path, self.config.default_pattern)
+            # Add archive prefix to list
+            self.config.archive_details[arch_url + "prefix"] = archive.prefix
+            self.prefixes[arch_url] = archive.prefix
+            # Add archive to list
+            src_objects.append(archive)
 
-    path = build.base_path + tarball_prefix
+        return src_objects
 
-    for archive, destination in zip(archives[::2], archives[1::2]):
+    def process(self, filemanager):
-        source_tarball_path = check_or_get_file(archive, os.path.basename(archive))
+        """Download and process the tarball."""
-        if source_tarball_path.lower().endswith('.zip'):
+        # determine build pattern and build requirements from url
-            tarball_contents = subprocess.check_output(
+        self.set_giturl_and_domain()
-                ["unzip", "-l", source_tarball_path], universal_newlines=True)
+        # determine name and version of package
-            if tarball_contents and len(tarball_contents.splitlines()) > 3:
+        self.name_and_version(filemanager)
-                source_tarball_prefix = tarball_contents.splitlines()[3].rsplit("/")[0].split()[-1]
+        # set gcov file information, must be done after name is set since the gcov
-            extract_cmd = "unzip -d {0} {1}".format(build.base_path, source_tarball_path)
+        # name is created by adding ".gcov" to the package name (if a gcov file
-        else:
+        # exists)
-            extract_cmd, source_tarball_prefix = build_untar(source_tarball_path)
+        self.set_gcov()
-        buildpattern.archive_details[archive + "prefix"] = source_tarball_prefix
+        # Download and process main source
-        call(extract_cmd)
+        main_src = self.process_main_source(self.url)
-        tar_files = glob.glob("{0}{1}/*".format(build.base_path, source_tarball_prefix))
+        # Store the detected prefix associated with this file
-        move_cmd = "mv "
+        self.prefixes[self.url] = main_src.prefix
-        for tar_file in tar_files:
+        self.tarball_prefix = main_src.prefix
-            move_cmd += tar_file + " "
+        # set global path with tarball_prefix
-        move_cmd += '{0}/{1}'.format(path, destination)
+        self.path = os.path.join(self.base_path, self.tarball_prefix)
-
+        # Now that the metadata has been collected print the header
-        mkdir_cmd = "mkdir -p "
+        self.print_header()
-        mkdir_cmd += '{0}/{1}'.format(path, destination)
+        # Download and process extra sources: archives
-
+        archives_src = self.process_archives()
-        print("mkdir " + mkdir_cmd)
+        # Extract all sources
-        call(mkdir_cmd)
+        self.extract_sources(main_src, archives_src)
-        call(move_cmd)
+        # Download and process any auto-generated source-tree archive for comparison
-
+        autogenerated_src = self.process_autogenerated_source(self.autogenerated_tarball)
-        sha1 = get_sha1sum(source_tarball_path)
+        # Extract autogenerated source for comparison
-        with open(build.download_path + "/upstream", "a") as file:
+        if autogenerated_src:
-            file.write(sha1 + "/" + os.path.basename(archive) + "\n")
+            autogenerated_src.extract(os.path.join(self.base_path, 'autogenerated-tmp'))
-
+            # Move the autogenerated source to a non-version-named directory for consistent diffs
-
+            call(f"mv autogenerated-tmp/{autogenerated_src.prefix} autogenerated", check=True, cwd=self.base_path)
-def write_nvr(file):
+            call("diff -u -r --unidirectional-new-file ../autogenerated ./",
-    global name
+                 logfile="archive.diff.in", check=False, cwd=os.path.join(self.base_path, main_src.prefix))
-    global version
+            call("grep -A14 -E '^(diff|Only in)' archive.diff.in",
-    global release
+                 logfile="archive.diff", check=False, cwd=os.getcwd())
-    file.write("Name     : " + name + "\n")
+            call("rm archive.diff.in", check=False, cwd=os.getcwd())
-    file.write("Version  : " + version + "\n")
-    file.write("Release  : " + str(release) + "\n")
-    file.write("URL      : " + url + "\n")
-    file.write("Source0  : " + url + "\n")

autospec/test.py

@@ -1,167 +0,0 @@
-#!/bin/true
-#
-# test.py - part of autospec
-# Copyright (C) 2015 Intel Corporation
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Deduce and emmit the patterns for %check
-#
-
-import buildpattern
-import buildreq
-import glob
-import os
-import tarball
-import subprocess
-import config
-
-tests_config = ""
-skip_tests = False
-allow_test_failures = False
-new_pkg = True
-unit_pass_written = False
-
-
-def check_regression(dir):
-
-    global skip_tests
-    if skip_tests:
-        return
-
-    build_log_path = os.path.join(dir, "results/build.log")
-    perl_cmd = ["perl", os.path.dirname(__file__) + "/count.pl", build_log_path]
-    result = subprocess.check_output(perl_cmd)
-    result = result.decode("utf-8")
-    lines = result.strip('\n').split('\n')
-    titles = ['package name', 'total tests', 'tests passing', 'tests failing',
-              'tests skipped', 'expected fail']
-    if len(lines) > 1:
-        for l in lines:
-            split_lines = l.split(',')
-            for x in range(0, len(split_lines)):
-                print(titles[x] + ": " + split_lines[x])
-    else:
-        split_list = lines[0].split(',')
-        for x in range(1, len(split_list)):
-            print(titles[x] + ": " + split_list[x])
-
-
-def scan_for_tests(dir):
-    global tests_config
-    global skip_tests
-    global allow_test_failures
-
-    if skip_tests:
-        return
-
-    if len(tests_config) > 0:
-        return
-
-    makeflags = "%{?_smp_mflags} " if config.parallel_build else ""
-    testsuites = {
-        "makecheck": "make VERBOSE=1 V=1 {}check".format(makeflags),
-        "perlcheck": "make TEST_VERBOSE=1 test",
-        "setup.py": "PYTHONPATH=%{buildroot}/usr/lib/python2.7/site-packages python2 setup.py test",
-        "cmake": "pushd clr-build ; make test ; popd",
-        "rakefile": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrake --trace test TESTOPTS=\"-v\"\npopd",
-        # "rubygems": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nruby -I\"lib:test*\" test*/*_test.rb \nruby -I\"lib:test*\" test*/test_*.rb\npopd",
-        "rspec": "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix + "\nrspec -I.:lib spec/\npopd"
-    }
-
-    files = os.listdir(dir)
-
-    if "CMakeLists.txt" in files:
-        makefile_path = os.path.join(dir, "CMakeLists.txt")
-        if not os.path.isfile(makefile_path):
-            return
-        with open(makefile_path, encoding="latin-1") as fp:
-            lines = fp.readlines()
-        for line in lines:
-            if line.find("enable_testing") >= 0:
-                tests_config = testsuites["cmake"]
-                break
-
-    if "Makefile.in" in files:
-        makefile_path = os.path.join(dir, "Makefile.in")
-        if not os.path.isfile(makefile_path):
-            return
-        with open(makefile_path) as fp:
-            lines = fp.readlines()
-        for line in lines:
-            if line.startswith("check:"):
-                tests_config = testsuites["makecheck"]
-                break
-            if line.startswith("test:"):
-                tests_config = testsuites["perlcheck"]
-                break
-    elif "Makefile.am" in files:
-        tests_config = testsuites["makecheck"]
-    elif tarball.name.startswith("rubygem"):
-        if "test" in files or "tests" in files:
-            r_testdir = [f for f in files if "test" in f if "." not in f].pop()
-            pre_test = glob.glob(os.path.join(dir, "test*/test_*.rb"))
-            post_test = glob.glob(os.path.join(dir, "test*/*_test.rb"))
-            tests_config = "pushd %{buildroot}%{gem_dir}/gems/" + tarball.tarball_prefix
-            if len(pre_test) > 0:
-                tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/test_*.rb"
-            if len(post_test) > 0:
-                tests_config += "\nruby -v -I.:lib:" + r_testdir + " test*/*_test.rb"
-            tests_config += "\npopd"
-
-        elif "spec" in files:
-            buildreq.add_buildreq("rubygem-rspec")
-            buildreq.add_buildreq("rubygem-rspec-core")
-            buildreq.add_buildreq("rubygem-rspec-expectations")
-            buildreq.add_buildreq("rubygem-rspec-support")
-            buildreq.add_buildreq("rubygem-rspec-mocks")
-            buildreq.add_buildreq("rubygem-devise")
-            buildreq.add_buildreq("rubygem-diff-lcs")
-            tests_config = testsuites["rspec"]
-        elif "Rakefile" in files:
-            with open(os.path.join(dir, "Rakefile"), encoding="ascii", errors="surrogateescape") as fp:
-                setup_contents = fp.read()
-                # if setup_contents.find("task :test") >= 0 or setup_contents.find("task 'test'") >= 0:
-                tests_config = testsuites["rakefile"]
-                buildreq.add_buildreq("ruby")
-                buildreq.add_buildreq("rubygem-rake")
-                buildreq.add_buildreq("rubygem-test-unit")
-                buildreq.add_buildreq("rubygem-minitest")
-    elif "Makefile.PL" in files:
-        tests_config = testsuites["perlcheck"]
-    elif "setup.py" in files:
-        with open(os.path.join(dir, "setup.py"), encoding="ascii", errors="surrogateescape") as fp:
-            setup_contents = fp.read()
-            if "test_suite" in setup_contents or "pbr=True" in setup_contents:
-                tests_config = testsuites["setup.py"]
-    elif buildpattern.default_pattern == "R":
-        tests_config = "export _R_CHECK_FORCE_SUGGESTS_=false\nR CMD check --no-manual --no-examples --no-codoc -l %{buildroot}/usr/lib64/R/library " + tarball.rawname
-
-    if "tox.ini" in files:
-        buildreq.add_buildreq("tox")
-        buildreq.add_buildreq("pytest")
-        buildreq.add_buildreq("virtualenv")
-        buildreq.add_buildreq("pluggy")
-        buildreq.add_buildreq("py-python")
-
-    if tests_config != "" and allow_test_failures:
-        if tarball.name.startswith("rubygem"):
-            config_elements = tests_config.split("\n")
-            config_elements.pop()
-            tests_config = "\n".join(config_elements) + " || :\npopd"
-        elif tests_config == testsuites["cmake"]:
-            tests_config = "pushd clr-build ; make test ||: ; popd"
-        else:
-            tests_config = tests_config + " || :"
-        print(tests_config)

autospec/translate.dic

@@ -1,54 +1,82 @@
-babel=Babel
+async_timeout=pypi-async_timeout
+babel=pypi-babel
+backports.ssl-match-hostname=pypi-backports.ssl_match_hostname
 barbicanclient-python=python-barbicanclient
-BeautifulSoup=beautifulsoup4
+BeautifulSoup=pypi-beautifulsoup4
-cryptography-python=cryptography
+cryptography-python=pypi-cryptography
 Crypto-python=pycrypt-python
-dateutil.relativedelta-python=python-dateutil
+dateutil-python=pypi-python_dateutil
+dateutil.relativedelta-python=pypi-python_dateutil
 designateclient-python=python-designateclient
-django-python=Django
+django-python=pypi-django
-enum-python=enum34-python
+enum-python=enum34
 formencode=FormEncode
 Genshi-python=Genshi
+gi.overrides-python=pygobject-python
+gi-python=pygobject-python
+github.com/golang/glog=golang-github-golang-glog
+github.com/gorilla/context=golang-github-gorilla-context
+github.com/mattn/go-sqlite3=golang-github-mattn-go-sqlite3
+github.com/mitchellh/mapstructure=golang-github-mitchellh-mapstructure
+github.com/stretchr/testify/assert=golang-github-stretchr-testify
+glance-store=glance_store
+golang.org/x/net/context=golang-googlecode-go-net
+golang.org/x/net/netutil=golang-googlecode-go-net
+gopkg.in/check.v1=golang-github-go-check-check
+gopkg.in/yaml.v2=golang-github-go-yaml-yaml
 hacking-python=hacking
 httplib2-python=httplib2
 ironicclient-python=python-ironicclient
-jinja2-python=Jinja2
+jinja2=pypi-jinja2
-Jinja2-python=Jinja2
+jinja2-python=pypi-jinja2
+Jinja2-python=pypi-jinja2
+jsonpath_rw_ext=jsonpath-rw-ext
 keystoneclient.auth-python=python-keystoneclient
 keystoneclient.v3-python=python-keystoneclient
 keystonemiddleware-python=keystonemiddleware
 lazy_object_proxy=lazy-object-proxy
 manilaclient-python=python-manilaclient
 mistralclient.api.v2-python=python-mistralclient
-mock=python-mock
 netaddr-python=netaddr
+Opcodes=opcodes
 openstackclient.tests-python=python-openstackclient
 openstack.nose-plugin-python=openstack.nose_plugin
 os_brick.initiator-python=os-brick
 oslo_cache-python=oslo.cache
-oslo.config-python=oslo.config
 oslo_config-python=oslo.config
+oslo.config-python=oslo.config
 oslo_policy-python=oslo.policy
 oslo_reports-python=oslo.reports
 oslo_versionedobjects-python=oslo.versionedobjects
 oslo_versionedobjects.tests-python=oslo.versionedobjects
-pep8-python=pep8
+pep8-python=pypi-pep8
-posix-ipc-python=posix_ipc
+posix-ipc-python=pypi-posix_ipc
-PrettyTable-python=prettytable
+PrettyTable-python=pypi-prettytable
-pyaml-python=pyaml
+pyaml-python=pypi-pyaml
 PyECLib-python=pyeclib
-Pygments-python=Pygments
+pygments=pypi-pygments
+Pygments-python=pypi-pygments
+PyGObject=pygobject
 pylibmc-python=pylibmc
+PyOpenSSL=pypi-pyopenssl
+pyqt5=PyQt5
+pytest_runner=pytest-runner
 python-memcached-python=python-memcached
-setuptools-scm-python=setuptools_scm
+pyyaml=PyYAML
+qtawesome=QtAwesome
+qtpy=QtPy
+restructuredtext-lint=restructuredtext_lint
+semantic-version=semantic_version
+setuptools-scm-python=pypi-setuptools_scm
 simplejson-python=simplejson
-sphinx=Sphinx
+sphinx=pypi-sphinx
-sqlparse-python=sqlparse
+sqlalchemy=pypi-sqlalchemy
+sqlparse-python=pypi-sqlparse
 stevedore-python=stevedore
 testresources-python=testresources
 testscenarios-python=testscenarios
 websocket-client=websocket_client
-WebTest-python=WebTest
+WebTest-python=pypi-webtest
 yaml-python=PyYAML
 zaqarclient.queues.v1-python=python-zaqarclient
 zaqarclient.transport-python=python-zaqarclient

autospec/util.py

@@ -17,15 +17,20 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import hashlib
 import os
+import re
 import shlex
 import subprocess
+import sys
 
 dictionary_filename = os.path.dirname(__file__) + "/translate.dic"
 dictionary = [line.strip() for line in open(dictionary_filename, 'r')]
+os_paths = None
 
 
 def call(command, logfile=None, check=True, **kwargs):
+    """Subprocess.call convenience wrapper."""
     returncode = 1
     full_args = {
         "args": shlex.split(command),
@@ -55,8 +60,110 @@ def _file_write(self, s):
 
 
 def translate(package):
+    """Convert terms to their alternate definition."""
     global dictionary
     for item in dictionary:
         if item.startswith(package + "="):
             return item.split("=")[1]
     return package
+
+
+def do_regex(patterns, re_str):
+    """Find a match in multiple patterns."""
+    for p in patterns:
+        match = re.search(p, re_str)
+        if match:
+            return match
+
+
+def get_contents(filename):
+    """Get contents of filename."""
+    with open(filename, "rb") as f:
+        return f.read()
+    return None
+
+
+def get_sha1sum(filename):
+    """Get sha1 sum of filename."""
+    sh = hashlib.sha1()
+    sh.update(get_contents(filename))
+    return sh.hexdigest()
+
+
+def _supports_color():
+    # FIXME: check terminfo instead
+    return sys.stdout.isatty()
+
+
+def _print_message(message, level, color=None):
+    prefix = level
+    if color and _supports_color():
+        # FIXME: use terminfo instead
+        if color == 'red':
+            params = '31;1'
+        elif color == 'green':
+            params = '32;1'
+        elif color == 'yellow':
+            params = '33;1'
+        elif color == 'blue':
+            params = '34;1'
+        prefix = f'\033[{params}m{level}\033[0m'
+    print(f'[{prefix}] {message}')
+
+
+def print_error(message):
+    """Print error, color coded for TTYs."""
+    _print_message(message, 'ERROR', 'red')
+
+
+def print_fatal(message):
+    """Print fatal error, color coded for TTYs."""
+    _print_message(message, 'FATAL', 'red')
+
+
+def print_warning(message):
+    """Print warning, color coded for TTYs."""
+    _print_message(message, 'WARNING', 'red')
+
+
+def print_info(message):
+    """Print informational message, color coded for TTYs."""
+    _print_message(message, 'INFO', 'yellow')
+
+
+def print_success(message):
+    """Print success message, color coded for TTYs."""
+    _print_message(message, 'SUCCESS', 'green')
+
+
+def binary_in_path(binary):
+    """Determine if the given binary exists in the provided filesystem paths."""
+    global os_paths
+    if not os_paths:
+        os_paths = os.getenv("PATH", default="/usr/bin:/bin").split(os.pathsep)
+
+    for path in os_paths:
+        if os.path.exists(os.path.join(path, binary)):
+            return True
+    return False
+
+
+def write_out(filename, content, mode="w"):
+    """File.write convenience wrapper."""
+    if mode:
+        with open_auto(filename, mode) as require_f:
+            require_f.write(content)
+
+
+def open_auto(*args, **kwargs):
+    """Open a file with UTF-8 encoding.
+
+    Open file with UTF-8 encoding and "surrogate" escape characters that are
+    not valid UTF-8 to avoid data corruption.
+    """
+    # 'encoding' and 'errors' are fourth and fifth positional arguments, so
+    # restrict the args tuple to (file, mode, buffering) at most
+    assert len(args) <= 3
+    assert 'encoding' not in kwargs
+    assert 'errors' not in kwargs
+    return open(*args, encoding="utf-8", errors="surrogateescape", **kwargs)

requirements.txt

@@ -0,0 +1,6 @@
+flake8>=3.4.0
+pycurl>=7.43.0
+toml>=0.9.0
+mock>=2.0.0
+coverage>=4.4.1
+requests>=2.18.4

setup.cfg

@@ -1,2 +1,12 @@
 [egg_info]
 tag_build = 
+[pycodestyle]
+ignore = E501
+
+[coverage:run]
+# omit tests and site-packages content
+omit = tests/*,*site-packages*,*site.py
+
+[flake8]
+max-line-length = 199
+ignore = B902, E722, W503

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.0.0"
+version = "1.2.0"
 
 def readme():
     with open("README.rst") as f:
@@ -21,10 +21,13 @@ setup(name="autospec",
             'Topic :: Software Development :: Build Tools',
             'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
             # Python versions supported.
+            'Programming Language :: Python :: 3 :: Only',
             'Programming Language :: Python :: 3',
             'Programming Language :: Python :: 3.2',
             'Programming Language :: Python :: 3.3',
             'Programming Language :: Python :: 3.4',
+            'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
         ],
       include_package_data = True,
 )

tests/COPYING_TEST

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

tests/README.rst

@@ -0,0 +1,23 @@
+================
+Autospec Testing
+================
+
+Code Style
+==========
+
+Autospec changes are scanned to vet code style issues with the ``flake8`` tool.
+To check for issues, run ``make check`` from the root of the autospec source
+tree, which executes ``flake8`` with appropriate arguments.
+
+Unit
+====
+
+Autospec ships with several test modules that correspond to individual modules
+from the toplevel ``autospec`` directory.
+
+Each module can be tested in isolation by running ``make test_<MODULE>``, where
+``<MODULE>`` corresponds to the module name. For example, ``make
+test_pkg_integrity`` runs unit tests for the ``pkg_integrity.py`` module.
+
+To run *all* unit tests, run ``make unittests``. If all tests pass, a code
+coverage report is also generated.

tests/builderrors

@@ -0,0 +1,103 @@
+    !  swig is not installed|perl(swig)
+    ExtUtils::Depends not installed|perl(ExtUtils::Depends)
+  Could not find swig|swig
+  Did not find swig|swig
+ exec: swig: not found|swig
+ mate-common.m4 not found|mate-common-dev
+-- Could NOT find swig|swig
+-- Could not find wpa_supplicant|wpa_supplicant
+-- Could not find xdg-open|xdg-utils
+-- swig not found.|swig
+etc etc /usr/bin/swig   not found|swig
+/bin/ld: cannot find -lz|zlib-dev
+/usr/bin/env: swig: No such file or directory|swig
+/usr/bin/python: No module named swig|swig
+Add the installation prefix of "swig" to CMAKE_PREFIX_PATH|swig
+By not providing "Findswig.cmake" in CMAKE_MODULE_PATH this|swig
+By not providing "swig.cmake" in CMAKE_MODULE_PATH this project|swig
+C library 'swig' not found|swig
+CMake Error at cmake/modules/swig.cmake|swig
+Can't locate swig in @INC (you may need to install the swig module)|perl(swig)
+Cannot find swig|swig
+Checking for swig     : not found|swig
+Checking for swig  >=    1.2   : not found|swig
+Checking for swig >= 1.2   : not found|swig
+Checking for swig development files... No|swig
+Checking for swig...no|swig
+Checking for swig: not found|swig
+Could NOT find swig|swig
+Could not find wpa_supplicant|wpa_supplicant
+Could not find xdg-open|xdg-utils
+Could not find a package configuration file provided by "swig" etc etc|swig
+Could not find suitable distribution for Requirement.parse('swig')|swig
+Dependency swig found: NO|swig
+Dependency testpkg found: NO (tried pkgconfig and cmake)|pkgconfig(testpkg)
+Dependency testpkg found: NO (tried pkgconfig)|pkgconfig(testpkg)
+Download error on https://pypi.python.org/simple/swig/|pypi(swig)
+Downloading https://test.python.org/packages/pkg/t/swig/1.23|swig
+ERROR: dependencies 'swig' are not available for package 'something'|R-swig
+ERROR: dependencies 'swig', 'swig2' are not available for package 'something'|R-swig
+ERROR: dependencies ‘swig’ are not available for package ‘something’|R-swig
+ERROR: dependencies ‘swig’, ‘swig2’ are not available for package ‘something’|R-swig
+ERROR: dependency 'swig' is not available for package 'something'|R-swig
+ERROR: dependency ‘swig’ is not available for package ‘something’|R-swig
+Error: Unable to find swig|swig
+Error: package 'swig' required by|R-swig
+Error: package ‘swig’ required by|R-swig
+ImportError: No module named 'swig'|pypi(swig)
+ImportError: No module named swig|swig
+ImportError: No module named swig|pypi(swig)
+ImportError: cannot import name swig|swig
+ImportError: swig module missing|swig
+ImportError:..*: No module named swig|pypi(swig)
+ModuleNotFoundError: No module named swig|pypi(swig)
+ModuleNotFoundError: No module named 'foo-bar'|pypi(foo_bar)
+Native dependency 'swig' not found|pkgconfig(swig)
+No local packages or working download links found for swig|pypi(swig)
+No matching distribution found for swig|pypi(swig)
+No package 'swig' found|pkgconfig(swig)
+No rule to make target `swig', etc|swig
+Package 'swig', required by 'something', not found|pkgconfig(swig)
+Package which this enhances but not available for checking: 'swig'|R-swig
+Package which this enhances but not available for checking: ‘swig’|R-swig
+Perhaps you should add the directory containing `swig.pc'|pkgconfig(swig)
+Program swig found: NO|swig
+Target 't' can't be generated as 'swig' could not be found|swig
+Unable to `import swig`|swig
+Unable to find 'swig'|swig
+Warning: no usable swig found|swig
+Warning: prerequisite swig 8 not found.|perl(swig)
+You need swig to build this program.|swig
+"swig" with any of the following names|swig
+checking for library containing swig... no|swig
+checking for perl module swig 7... no|perl(swig)
+checking for something in swig... no|swig
+checking for swig in default path... not found|swig
+checking for swig support... no|swig
+checking for swig support... no|swig
+checking for swig with pkg-config... no|swig
+checking for swig... configure: error|swig
+checking for swig... no|swig
+checking for swig... not found|swig
+checking for swig... not_found|swig
+checking swig... no|swig
+configure: error: Cannot find swig. Make sure|swig
+configure: error: Unable to locate swig|swig
+configure: error: pkg-config missing swig|swig
+configure: error: swig is required to build|swig
+configure: error: swig not found|swig
+dependency testpkg found: NO (tried pkgconfig and cmake)|pkgconfig(testpkg)
+dependency testpkg found: NO (tried pkgconfig)|pkgconfig(testpkg)
+fatal error: swig: No such file or directory|swig
+make: swig: Command not found|swig
+make: help2man: No such file or directory|help2man
+swig 8 is required to configure this module; please install it or upgrade your CPAN/CPANPLUS shell.|swig
+swig tool not found or not executable|swig
+swig validation tool not found or not executable|swig
+swig: command not found|swig
+there is no package called 'swig'|R-swig
+there is no package called ‘swig’|R-swig
+unable to execute 'swig': No such file or directory|swig
+warning: failed to load external entity "/usr/share/sgml/docbook/xsl-stylesheets/something"|docbook-xml
+which: no swig in (/usr/bin/swig/)|swig
+you may need to install the swig module|perl(swig)

tests/test_abireport.py

@@ -0,0 +1,273 @@
+import unittest
+import abireport
+
+
+def mock_return(retval):
+    """
+    Simple mock method to set return value of a function
+    """
+    def mock_fn(_):
+        return retval
+
+    return mock_fn
+
+
+class TestAbireport(unittest.TestCase):
+
+    def test_get_soname(self):
+        """
+        Test get_soname function with valid get_output results
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return('SONAME               libtest.so.14')
+        self.assertEqual(abireport.get_soname('test'), 'libtest.so.14')
+        abireport.get_output = backup
+
+    def test_get_soname_none(self):
+        """
+        Test get_soname function with no get_output results
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return('')
+        self.assertEqual(abireport.get_soname('test'), None)
+        abireport.get_output = backup
+
+    def test_get_soname_error(self):
+        """
+        Test get_soname function with exception raised in get_output
+        """
+        backup = abireport.get_output
+
+        def mock_get_output(cmd):
+            raise Exception('testing exception')
+
+        abireport.get_output = mock_get_output
+        self.assertEqual(abireport.get_soname('test'), None)
+        abireport.get_output = backup
+
+    def test_get_shared_dependencies(self):
+        """
+        Test get_shared_dependencies function with valid get_output results
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return(READELF1)
+        results = set(['libdl.so.2', 'libz.so.1', 'libexpat.so.1',
+                       'libstdc++.so.6', 'libm.so.6', 'libgcc_s.so.1',
+                       'libc.so.6'])
+        self.assertEqual(abireport.get_shared_dependencies('test'), results)
+        abireport.get_output = backup
+
+    def test_get_shared_dependencies_none(self):
+        """
+        Test get_shared_dependencies with no matches
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return(READELF2)
+        self.assertEqual(abireport.get_shared_dependencies('test'), set())
+        abireport.get_output = backup
+
+    def test_is_dynamic_binary(self):
+        """
+        Test is_dynamic_binary function for True return
+        """
+        backup_isfile = abireport.os.path.isfile
+        backup_exists = abireport.os.path.exists
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.isfile = mock_return(True)
+        abireport.os.path.exists = mock_return(True)
+        abireport.get_file_magic = mock_return(FILEMAGIC1)
+
+        self.assertTrue(abireport.is_dynamic_binary('test'))
+
+        abireport.os.path.isfile = backup_isfile
+        abireport.os.path.exists = backup_exists
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_is_dynamic_binary_false(self):
+        """
+        Test is_dynamic_binary function for False return
+        """
+        backup_exists = abireport.os.path.exists
+        backup_isfile = abireport.os.path.isfile
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.exists = mock_return(True)
+        abireport.os.path.isfile = mock_return(True)
+        abireport.get_file_magic = mock_return('This is not a matched string')
+
+        self.assertFalse(abireport.is_dynamic_binary('test'))
+
+        abireport.os.path.exists = backup_exists
+        abireport.os.path.isfile = backup_isfile
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_is_file_valid(self):
+        """
+        Test is_file_valid function with valid file magic
+        """
+        backup_exists = abireport.os.path.exists
+        backup_islink = abireport.os.path.islink
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.exists = mock_return(True)
+        abireport.os.path.islink = mock_return(False)
+        abireport.get_file_magic = mock_return(FILEMAGIC1)
+
+        self.assertTrue(abireport.is_file_valid('test'))
+
+        abireport.os.path.exists = backup_exists
+        abireport.os.path.islink= backup_islink
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_is_file_valid_false(self):
+        """
+        Test is_file_valid function with valid file magic
+        """
+        backup_exists = abireport.os.path.exists
+        backup_islink = abireport.os.path.islink
+        backup_get_file_magic = abireport.get_file_magic
+
+        abireport.os.path.exists = mock_return(True)
+        abireport.os.path.islink = mock_return(False)
+        abireport.get_file_magic = mock_return('This is not a matched string')
+
+        self.assertFalse(abireport.is_file_valid('test'))
+
+        abireport.os.path.exists = backup_exists
+        abireport.os.path.islink= backup_islink
+        abireport.get_file_magic = backup_get_file_magic
+
+    def test_dump_symbols(self):
+        """
+        Test dump_symbols function with valid nm output
+        """
+        backup = abireport.get_output
+        abireport.get_output = mock_return(NM1)
+        results = set(['WXMPIterator_DecrementRefCount_1',
+                       'WXMPIterator_IncrementRefCount_1',
+                       '_Z13XMP_InitMutexP15pthread_mutex_t',
+                       '_Z13XMP_TermMutexR15pthread_mutex_t',
+                       '_Z14CloneOffspringPK8XMP_NodePS_',
+                       '_Z14SortNamedNodesRSt6vectorIP8XMP_NodeSaIS1_EE',
+                       '_Z15CompareSubtreesRK8XMP_NodeS1_',
+                       '_ZN5Exiv211focalLengthERKNS_8ExifDataE',
+                       '_ZN5Exiv211getRationalEPKhNS_9ByteOrderE',
+                       '_ZN5Exiv211isBigEndianEv',
+                       '_ZN5Exiv211orientationERKNS_8ExifDataE',
+                       '_ZN5Exiv28GifImageC2ESt8auto_ptrINS_7BasicIoEE'])
+        self.assertEqual(abireport.dump_symbols('test'), results)
+        abireport.get_output = backup
+
+    def test_dump_symbols_exit(self):
+        """
+        Test dump_symbols function with fatal exception
+        """
+        backup = abireport.get_output
+
+        def mock_get_output(_):
+            raise Exception('This is a test exception')
+
+        abireport.get_output = mock_get_output
+        with self.assertRaises(SystemExit) as dumpsymbols:
+            abireport.dump_symbols('test')
+
+        self.assertEqual(dumpsymbols.exception.code, 1)
+
+
+READELF1 = """
+
+Dynamic section at offset 0x2d0788 contains 31 entries:
+  Tag        Type                         Name/Value
+ 0x0000000000000001 (NEEDED)             Shared library: [libdl.so.2]
+ 0x0000000000000001 (NEEDED)             Shared library: [libz.so.1]
+ 0x0000000000000001 (NEEDED)             Shared library: [libexpat.so.1]
+ 0x0000000000000001 (NEEDED)             Shared library: [libstdc++.so.6]
+ 0x0000000000000001 (NEEDED)             Shared library: [libm.so.6]
+ 0x0000000000000001 (NEEDED)             Shared library: [libgcc_s.so.1]
+ 0x0000000000000001 (NEEDED)             Shared library: [libc.so.6]
+ 0x000000000000000e (SONAME)             Library soname: [libexiv2.so.14]
+ 0x000000000000000f (RPATH)              Library rpath: [/usr/lib]
+ 0x000000000000000c (INIT)               0xca320
+ 0x000000000000000d (FINI)               0x20e6cc
+ 0x0000000000000019 (INIT_ARRAY)         0x2a5a70
+ 0x000000000000001b (INIT_ARRAYSZ)       400 (bytes)
+ 0x000000000000001a (FINI_ARRAY)         0x2a5c00
+ 0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
+ 0x0000000000000004 (HASH)               0x1f0
+ 0x0000000000000005 (STRTAB)             0x23f10
+ 0x0000000000000006 (SYMTAB)             0x8ac0
+ 0x000000000000000a (STRSZ)              252569 (bytes)
+ 0x000000000000000b (SYMENT)             24 (bytes)
+ 0x0000000000000003 (PLTGOT)             0x2d19b8
+ 0x0000000000000007 (RELA)               0x63f48
+ 0x0000000000000008 (RELASZ)             418776 (bytes)
+ 0x0000000000000009 (RELAENT)            24 (bytes)
+ 0x0000000000000018 (BIND_NOW)           
+ 0x000000006ffffffb (FLAGS_1)            Flags: NOW
+ 0x000000006ffffffe (VERNEED)            0x63e08
+ 0x000000006fffffff (VERNEEDNUM)         5
+ 0x000000006ffffff0 (VERSYM)             0x619aa
+ 0x000000006ffffff9 (RELACOUNT)          11626
+ 0x0000000000000000 (NULL)               0x0
+"""
+
+READELF2 = """
+
+Dynamic section at offset 0x2d0788 contains 31 entries:
+  Tag        Type                         Name/Value
+ 0x000000000000000e (SONAME)             Library soname: [libexiv2.so.14]
+ 0x000000000000000f (RPATH)              Library rpath: [/usr/lib]
+ 0x000000000000000c (INIT)               0xca320
+ 0x000000000000000d (FINI)               0x20e6cc
+ 0x0000000000000019 (INIT_ARRAY)         0x2a5a70
+ 0x000000000000001b (INIT_ARRAYSZ)       400 (bytes)
+ 0x000000000000001a (FINI_ARRAY)         0x2a5c00
+ 0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
+ 0x0000000000000004 (HASH)               0x1f0
+ 0x0000000000000005 (STRTAB)             0x23f10
+ 0x0000000000000006 (SYMTAB)             0x8ac0
+ 0x000000000000000a (STRSZ)              252569 (bytes)
+ 0x000000000000000b (SYMENT)             24 (bytes)
+ 0x0000000000000003 (PLTGOT)             0x2d19b8
+ 0x0000000000000007 (RELA)               0x63f48
+ 0x0000000000000008 (RELASZ)             418776 (bytes)
+ 0x0000000000000009 (RELAENT)            24 (bytes)
+ 0x0000000000000018 (BIND_NOW)           
+ 0x000000006ffffffb (FLAGS_1)            Flags: NOW
+ 0x000000006ffffffe (VERNEED)            0x63e08
+ 0x000000006fffffff (VERNEEDNUM)         5
+ 0x000000006ffffff0 (VERSYM)             0x619aa
+ 0x000000006ffffff9 (RELACOUNT)          11626
+ 0x0000000000000000 (NULL)               0x0
+"""
+
+FILEMAGIC1 = '/path/to/so/libtest.so.1: ELF 64-bit LSB shared object, ' \
+             'x86-64, version 1 (SYSV), dynamically linked, '           \
+             'BuildID[sha1]=c99853fe0c5d8f3c7550848edf25fe7782fb22c3, stripped'
+
+NM1 = """
+00000000002f1d88 B voidStringPtr
+00000000002f1d90 B voidVoidPtr
+00000000002f1ce0 B void_wResult
+00000000001d1a40 T WXMPIterator_DecrementRefCount_1
+00000000001d19f0 T WXMPIterator_IncrementRefCount_1
+00000000001daa80 T _Z13XMP_InitMutexP15pthread_mutex_t
+00000000001daaa0 T _Z13XMP_TermMutexR15pthread_mutex_t
+00000000001db900 T _Z14CloneOffspringPK8XMP_NodePS_
+00000000001df4e0 A _Z14SortNamedNodesRSt6vectorIP8XMP_NodeSaIS1_EE
+00000000001dc450 T _Z15CompareSubtreesRK8XMP_NodeS1_
+000000000011d260 A _ZN5Exiv211focalLengthERKNS_8ExifDataE
+00000000001b0510 T _ZN5Exiv211getRationalEPKhNS_9ByteOrderE
+0000000000197160 T _ZN5Exiv211isBigEndianEv
+000000000011c8e0 T _ZN5Exiv211orientationERKNS_8ExifDataE
+00000000002c7000 D _ZN5Exiv211sectionInfoE
+00000000002c45e0 D _ZN5Exiv212xmpXmpBJInfoE
+0000000000137660 T _ZN5Exiv28GifImageC2ESt8auto_ptrINS_7BasicIoEE
+00000000002658d8 R _ZN5Exiv28Internal10canonCfCfgE
+0000000000265960 R _ZN5Exiv28Internal10canonCsCfgE
+00000000002ebac0 B _ZN5Exiv28Internal14SigmaMakerNote8tagInfo_E
+"""
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_build.py

@@ -0,0 +1,498 @@
+import unittest
+import tempfile
+import os
+from unittest.mock import patch, mock_open, MagicMock
+import build
+import buildreq
+import config
+import files
+import tarball
+
+class TestBuildpattern(unittest.TestCase):
+
+    def test_simple_pattern_pkgconfig(self):
+        """
+        Test simple_pattern_pkgconfig with match
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern_pkgconfig('line to test for testpkg.xyz',
+                                     r'testpkg.xyz',
+                                     'testpkg',
+                                     False,
+                                     reqs)
+        self.assertIn('pkgconfig(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_simple_pattern_pkgconfig_32bit(self):
+        """
+        Test simple_pattern_pkgconfig with match and 32bit option set
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern_pkgconfig('line to test for testpkg.zyx',
+                                     r'testpkg.zyx',
+                                     'testpkgz',
+                                     True,
+                                     reqs)
+        self.assertIn('pkgconfig(32testpkgz)', reqs.buildreqs)
+        self.assertIn('pkgconfig(testpkgz)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_simple_pattern_pkgconfig_no_match(self):
+        """
+        Test simple_pattern_pkgconfig with no match, nothing should be modified
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern_pkgconfig('line to test for somepkg.xyz',
+                                     r'testpkg.xyz',
+                                     'testpkg',
+                                     False,
+                                     reqs)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_simple_pattern(self):
+        """
+        Test simple_pattern with match. The main difference between
+        simple_pattern and simple_pattern_pkgconfig is the string that is added
+        to buildreq.buildreqs.
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern('line to test for testpkg.xyz',
+                           r'testpkg.xyz',
+                           'testpkg',
+                           reqs)
+        self.assertIn('testpkg', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_simple_pattern_no_match(self):
+        """
+        Test simple_pattern with no match, nothing should be modified
+        """
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.simple_pattern('line to test for somepkg.xyz',
+                           r'testpkg.xyz',
+                           'testpkg',
+                           reqs)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_failed_pattern_no_match(self):
+        """
+        Test failed_pattern with no match
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: somepkg', conf, reqs, r'(test)', 0)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_failed_pattern_no_buildtool(self):
+        """
+        Test failed_pattern with buildtool unset and initial match, but no
+        match in failed_commands.
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg', conf, reqs, r'(test)', 0)
+        self.assertEqual(reqs.buildreqs, set())
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_failed_pattern_no_buildtool_match(self):
+        """
+        Test failed_pattern with buildtool unset and match in failed_commands
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        conf.setup_patterns()
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: lex', conf, reqs, r'(lex)', 0)
+        self.assertIn('flex', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_pkgconfig(self):
+        """
+        Test failed_pattern with buildtool set to pkgconfig
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.xyz',
+                             conf,
+                             reqs,
+                             r'(testpkg)',
+                             0,  # verbose=0
+                             buildtool='pkgconfig')
+        self.assertIn('pkgconfig(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_R(self):
+        """
+        Test failed_pattern with buildtool set to R
+        """
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.r',
+                             conf,
+                             reqs,
+                             r'(testpkg)',
+                             0,  # verbose=0
+                             buildtool='R')
+        self.assertIn('R-testpkg', reqs.buildreqs)
+        self.assertNotIn('R-testpkg', reqs.requires[None])
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_perl(self):
+        """
+        Test failed_pattern with buildtool set to perl
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.pl',
+                             conf,
+                             reqs,
+                             r'(testpkg)',
+                             0,  # verbose=0
+                             buildtool='perl')
+        self.assertIn('perl(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_failed_pattern_pypi(self):
+        """
+        Test failed_pattern with buildtool set to pypi
+        """
+        conf = config.Config('')
+        reqs = buildreq.Requirements("")
+        pkg = build.Build()
+        pkg.failed_pattern('line to test for failure: testpkg.py',
+                           conf,
+                           reqs,
+                           r'(testpkg)',
+                           0,  # verbose=0
+                           buildtool='pypi')
+        self.assertIn('pypi(testpkg)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_buildroot_log_fail(self):
+        """
+        Test parse_buildroot_log with a test log indicating failure due to
+        missing dependencies ('foobar' and 'foobarbaz')
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+
+        open_name = 'build.util.open_auto'
+        content = "line1\nDEBUG util.py:399:  No matching package to install: 'foobar'\nDEBUG util.py:399:  No matching package to install: 'foobarbaz'\nline 4"
+        m_open = mock_open(read_data=content)
+        pkg = build.Build()
+        pkg.must_restart = 1
+        pkg.file_restart = 1
+
+        result = True
+        with patch(open_name, m_open, create=True):
+            result = pkg.parse_buildroot_log('testname', 1)
+
+        build.util.call = call_backup
+
+        self.assertFalse(result)
+        self.assertEqual(pkg.must_restart, 0)
+        self.assertEqual(pkg.file_restart, 0)
+
+    def test_parse_buildroot_log_pass(self):
+        """
+        Test parse_buildroot_log with a test log indicating no failures
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+
+        open_name = 'build.util.open_auto'
+        content = "line 1\nline 2\nline 3\nline 4"
+        m_open = mock_open(read_data=content)
+        pkg = build.Build()
+
+        result = True
+        with patch(open_name, m_open, create=True):
+            result = pkg.parse_buildroot_log('testname', 1)
+
+        build.util.call = call_backup
+
+        self.assertTrue(result)
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_parse_buildroot_log_noop(self):
+        """
+        Test parse_buildroot_log when parsing should be skipped (i.e. mock
+        returned 0)
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+
+        open_name = 'build.util.open_auto'
+        content = "line 1\nline 2\nline 3\nline 4"
+        m_open = mock_open(read_data=content)
+        pkg = build.Build()
+
+        result = True
+        with patch(open_name, m_open, create=True):
+            result = pkg.parse_buildroot_log('testname', 0)
+
+        build.util.call = call_backup
+
+        self.assertTrue(result)
+
+    def test_parse_build_results_patch(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        a backport patch no longer applying
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        def mock_conf_remove_backport_patch(patch):
+            del patch
+            return 1
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        conf.remove_backport_patch = mock_conf_remove_backport_patch
+        conf.patches = ['backport-test.patch']
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\nPatch #1 (backport-test.patch):\nSkipping patch.'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_build_results_pkgconfig(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        missing qmake package (pkgconfig error)
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        conf.config_opts['32bit'] = True
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\nwhich: no qmake\nexiting'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertIn('pkgconfig(Qt)', reqs.buildreqs)
+        self.assertIn('pkgconfig(32Qt)', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_build_results_simple_pats(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        missing httpd-dev package (simple pat error)
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\nchecking for Apache test module support\nexiting'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertIn('httpd-dev', reqs.buildreqs)
+        self.assertEqual(pkg.must_restart, 1)
+
+    def test_parse_build_results_failed_pats(self):
+        """
+        Test parse_build_results with a test log indicating failure due to a
+        missing package.
+        """
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        open_auto_backup = build.util.open_auto
+        build.util.call = MagicMock(return_value=None)
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        with open('tests/builderrors', 'r') as f:
+            builderrors = f.readlines()
+            for error in builderrors:
+                if not error.startswith('#'):
+                    input, output = error.strip('\n').split('|')
+                    reqs.buildreqs = set()
+                    build.util.open_auto = mock_open(read_data=input)
+                    pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+                    self.assertIn(output, reqs.buildreqs)
+                    self.assertGreater(pkg.must_restart, 0)
+
+        # Restoring functions
+        build.util.call = call_backup
+        build.util.open_auto = open_auto_backup
+
+    def test_parse_build_results_files(self):
+        """
+        Test parse_build_results with a test log indicating files are missing
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\n' \
+                  'Installed (but unpackaged) file(s) found:\n' \
+                  '/usr/testdir/file\n' \
+                  '/usr/testdir/file1\n' \
+                  '/usr/testdir/file2\n' \
+                  'RPM build errors\n' \
+                  'errors here\n'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertEqual(fm.files,
+                         set(['/usr/testdir/file',
+                              '/usr/testdir/file1',
+                              '/usr/testdir/file2']))
+        # one for each file added
+        self.assertEqual(pkg.must_restart, 0)
+        self.assertEqual(pkg.file_restart, 3)
+
+    def test_parse_build_results_banned_files(self):
+        """
+        Test parse_build_results with a test log indicating banned files are missing
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        conf = config.Config('')
+        conf.setup_patterns()
+        reqs = buildreq.Requirements("")
+        tcontent = tarball.Content("", "", "", [], conf, "/")
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        pkg = build.Build()
+        fm = files.FileManager(conf, pkg)
+
+        open_name = 'build.util.open_auto'
+        content = 'line 1\n' \
+                  'Installed (but unpackaged) file(s) found:\n' \
+                  '/opt/file\n' \
+                  '/usr/etc/file\n' \
+                  '/usr/local/file\n' \
+                  '/usr/src/file\n' \
+                  '/var/file\n' \
+                  'RPM build errors\n' \
+                  'errors here\n'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            pkg.parse_build_results('testname', 0, fm, conf, reqs, tcontent)
+
+        build.util.call = call_backup
+
+        self.assertEqual(fm.has_banned, True)
+        # check no files were added
+        self.assertEqual(pkg.must_restart, 0)
+
+    def test_get_mock_cmd_without_consolehelper(self):
+        """
+        Test get_mock_cmd when /usr/bin/mock doesn't point to consolehelper
+        """
+        def mock_realpath(path):
+            return path
+
+        realpath_backup = build.os.path.realpath
+
+        build.os.path.realpath = mock_realpath
+
+        mock_cmd = build.get_mock_cmd()
+
+        build.os.path.realpath = realpath_backup
+
+        self.assertEqual(mock_cmd, 'sudo /usr/bin/mock')
+
+    def test_get_mock_cmd_with_consolehelper(self):
+        """
+        Test get_mock_cmd when /usr/bin/mock points to consolehelper
+        """
+        def mock_realpath(path):
+            return '/usr/bin/consolehelper'
+
+        realpath_backup = build.os.path.realpath
+
+        build.os.path.realpath = mock_realpath
+
+        mock_cmd = build.get_mock_cmd()
+
+        build.os.path.realpath = realpath_backup
+
+        self.assertEqual(mock_cmd, '/usr/bin/mock')
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_buildreq.py

@@ -0,0 +1,839 @@
+import unittest
+import tempfile
+import os
+from unittest.mock import MagicMock, mock_open, patch
+import json
+import io
+import buildreq
+import config
+import pypidata
+
+bak_get_pypi_name = pypidata.get_pypi_name
+def get_pypi_name_wrapper(name, miss=None):
+    """Ignore missing packags in pypi"""
+    return bak_get_pypi_name(name)
+
+
+class TestBuildreq(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Test setup method to reset the buildreq module
+        """
+        self.reqs = buildreq.Requirements("")
+        self.reqs.banned_buildreqs.add('bannedreq')
+
+    def test_add_buildreq(self):
+        """
+        Test add_buildreq with unbanned new req. Follow up by asserting that
+        trying to add the same req a second time results in a False return.
+        """
+        self.assertTrue(self.reqs.add_buildreq('testreq'))
+        self.assertIn('testreq', self.reqs.buildreqs)
+        self.assertFalse(self.reqs.add_buildreq('testreq'))
+
+    def test_add_buildreq_banned(self):
+        """
+        Test add_buildreq with banned new req
+        """
+        self.assertFalse(self.reqs.add_buildreq('bannedreq'))
+        self.assertNotIn('bannedreq', self.reqs.buildreqs)
+
+    def test_ban_requires(self):
+        """
+        Test ban_requires with req already present in requires
+        """
+        self.reqs.requires[None] = set(['testreq'])
+        self.reqs.ban_requires('testreq')
+        self.assertNotIn('testreq', self.reqs.requires[None])
+
+    def test_ban_requires_subpkg(self):
+        """
+        Test ban_requires on a subpkg with req already present in requires
+        """
+        self.reqs.requires['subpkg'] = set(['testreq'])
+        self.reqs.ban_requires('testreq', subpkg='subpkg')
+        self.assertNotIn('testreq', self.reqs.requires['subpkg'])
+
+    def test_add_requires(self):
+        """
+        Test add_requires with unbanned new req already present in
+        buildreqs but not yet present in requires
+        """
+        self.reqs.add_buildreq('testreq')
+        self.assertTrue(self.reqs.add_requires('testreq', ['testreq']))
+        self.assertIn('testreq', self.reqs.requires[None])
+
+    def test_add_requires_subpkg(self):
+        """
+        Test add_requires on a subpkg with unbanned new req already present in
+        buildreqs but not yet present in requires
+        """
+        self.reqs.add_buildreq('testreq')
+        self.assertTrue(self.reqs.add_requires('testreq', ['testreq'], subpkg='subpkg'))
+        self.assertIn('testreq', self.reqs.requires['subpkg'])
+
+    def test_add_requires_not_in_buildreqs(self):
+        """
+        Test add_requires with unbanned new req not present in buildreqs.
+        """
+        self.assertFalse(self.reqs.add_requires('testreq', []))
+        self.assertNotIn('testreq', self.reqs.requires[None])
+
+    def test_add_banned_requires(self):
+        """
+        Test add_requires with banned new req (override buildreq).
+        """
+        self.assertFalse(self.reqs.add_requires('pypi(nose)', [], override=True))
+        self.assertNotIn('testreq', self.reqs.requires[None])
+
+    def test_ban_provides(self):
+        """
+        Test ban_provides with prov already present in provides
+        """
+        self.reqs.provides[None] = set(['testreq'])
+        self.reqs.ban_provides('testreq')
+        self.assertNotIn('testreq', self.reqs.provides[None])
+
+    def test_ban_provides_subpkg(self):
+        """
+        Test ban_provides on a subpkg with prov already present in provides
+        """
+        self.reqs.provides['subpkg'] = set(['testreq'])
+        self.reqs.ban_provides('testreq', subpkg='subpkg')
+        self.assertNotIn('testreq', self.reqs.provides['subpkg'])
+
+    def test_add_provides(self):
+        """
+        Test add_provides with unbanned new prov
+        """
+        self.assertTrue(self.reqs.add_provides('testreq'))
+        self.assertIn('testreq', self.reqs.provides[None])
+
+    def test_add_provides_subpkg(self):
+        """
+        Test add_provides on a subpkg with unbanned new prov
+        """
+        self.assertTrue(self.reqs.add_provides('testreq', subpkg='subpkg'))
+        self.assertIn('testreq', self.reqs.provides['subpkg'])
+
+    def test_add_pkgconfig_buildreq(self):
+        """
+        Test add_pkgconfig_buildreq with config_opts['32bit'] set to False
+        """
+        self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', False))
+        self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs)
+
+    def test_add_pkgconfig_buildreq_32bit(self):
+        """
+        Test add_pkgconfig_buildreq with config_opts['32bit'] set to True
+        """
+        self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', True))
+        self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs)
+        self.assertIn('pkgconfig(32testreq)', self.reqs.buildreqs)
+
+    def test_configure_ac_line(self):
+        """
+        Test configure_ac_line with standard pattern
+        """
+        self.reqs.configure_ac_line('AC_CHECK_FUNC([tgetent])', False)
+        self.assertIn('ncurses-devel', self.reqs.buildreqs)
+
+    def test_configure_ac_line_comment(self):
+        """
+        Test configure_ac_line with commented line
+        """
+        self.reqs.configure_ac_line('# AC_CHECK_FUNC([tgetent])', False)
+        self.assertEqual(self.reqs.buildreqs, set())
+
+    def test_configure_ac_line_pkg_check_modules(self):
+        """
+        Test the somewhat complicated logic of configure_ac_line check for the
+        PKG_CHECK_MODULES((.*?)) line.
+        """
+        self.reqs.configure_ac_line(
+            'PKG_CHECK_MODULES(prefix, '
+            '[module > 2 module2 < 2], '
+            'action-if-found, action-if-not-found)', False)
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(module)', 'pkgconfig(module2)']))
+
+    def test_configure_ac_line_xdt_check_package(self):
+        """
+        Test configure_ac_line for the XFCE version of PKG_CHECK_MODULES
+        """
+        self.reqs.configure_ac_line(
+            'XDT_CHECK_PACKAGE(prefix, '
+            '[module = 2 module2 > 9], '
+            'action-if-found, action-if-not-found)', False)
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(module)', 'pkgconfig(module2)']))
+
+    def test_configure_ac_line_pkg_check_exists(self):
+        """
+        Test configure_ac_line for the PKG_CHECK_EXISTS macro
+        """
+        self.reqs.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], '
+                                   'action-if-found, '
+                                   'action-if-not-found)', False)
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(module1)', 'pkgconfig(module2)']))
+
+    def test_parse_configure_ac(self):
+        """
+        Test parse_configure_ac with changing () depths and package
+        requirements
+        """
+        conf = config.Config("")
+        content = 'AC_CHECK_FUNC([tgetent])\n'                   \
+                  'XDT_CHECK_PACKAGE(prefix, '                   \
+                  '[module = 2 module2 > 9], '                   \
+                  'action-if-found, action-if-not-found)\n'      \
+                  'next two lines should be read in batch ( \n'  \
+                  'PROG_INTLTOOL\n'                              \
+                  'AC_PROG_SED)\n'                               \
+                  'GETTEXT_PACKAGE'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_configure_ac(os.path.join(tmpd, 'fname'), conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['gettext',
+                              'ncurses-devel',
+                              'perl(XML::Parser)',
+                              'pkgconfig(module2)',
+                              'pkgconfig(module)',
+                              'intltool',
+                              'sed']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_clean_python_req(self):
+        """
+        Test clean_python_req with a common python requirements string
+        """
+        self.assertEqual(buildreq.clean_python_req('requirement >= 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement ; python_version > 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement ; python_version < 1.1.2'),
+                         '')
+        self.assertEqual(buildreq.clean_python_req('requirement <= 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement = 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement \n ; rsa>= 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('requirement != 1.1.2'),
+                         'requirement')
+        self.assertEqual(buildreq.clean_python_req('[:python > 2]'),
+                         '')
+        self.assertEqual(buildreq.clean_python_req('requirement ~= 1.1.2'),
+                         'requirement')
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_clean_python_req_comment(self):
+        """
+        Test clean_python_req with a comment
+        """
+        self.assertEqual(buildreq.clean_python_req('# hello'), '')
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_clean_python_req_whitespace(self):
+        """
+        Test clean_python_req with strange whitespaced string
+        """
+        self.assertEqual(buildreq.clean_python_req('   requirement    < 1.1'),
+                        'requirement')
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_grab_python_requirements(self):
+        """
+        Test grab_python_requirements with a reasonable requirements file
+        """
+        # buildreqs must include the requires also
+        open_name = 'buildreq.util.open_auto'
+        content = 'req1 <= 1.2.3\n' \
+                  'req2 >= 1.55\n'  \
+                  'req7 == 3.3.3\n'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.grab_python_requirements('filename', [])
+
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_grab_python_requirements_strange_file(self):
+        """
+        Test grab_python_requirements with a poorly written file
+        """
+        # buildreqs must include the requires also
+        open_name = 'buildreq.util.open_auto'
+        content = '    req1 <= 1.2.3\n   ' \
+                  'req2    >= 1.55   \n'   \
+                  '   req7 == 3.3.3\n    '
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.grab_python_requirements('filename', [])
+
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires(self):
+        """
+        Test add_setup_py_requires with a single item in install_requires and
+        setup_requires
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=['req1']\n" \
+                  "setup_requires=['req2']"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline(self):
+        """
+        Test add_setup_py_requires with a multiline item in install_requires
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=['req1',\n" \
+                  "'req2',\n"                   \
+                  "'req7']\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline_formatted(self):
+        """
+        Test add_setup_py_requires with a multiline item in install_requires
+        with brackets on their own lines.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[\n "  \
+                  "'req1',\n"              \
+                  "'req2',\n"              \
+                  "'req7',\n"              \
+                  "]\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)', 'pypi(req7)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline_variable(self):
+        """
+        Test add_setup_py_requires with multiline item in install_requires that
+        contains a non-literal object.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[\n" \
+                  "reqvar,\n"            \
+                  "'req1',\n"            \
+                  "'req2'"               \
+                  "]\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_multiline_install_requires_variable(self):
+        """
+        Test add_setup_py_requires with multiline item in install_requires that
+        contains an extra bit of content that shouldn't be parsed as install_requires.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[\n"   \
+                  "'req1',\n"              \
+                  "'req2'"                 \
+                  "] + install_requires\n" \
+                  "'bad']\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_variable(self):
+        """
+        Test add_setup_py_requires that contains a non-literal object.
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires=[reqname, 'req1', 'req2']\n"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(req1)', 'pypi(req2)']))
+        self.assertEqual(self.reqs.requires["python3"], set(['pypi(req1)', 'pypi(req2)']))
+
+    @patch('buildreq.pypidata.get_pypi_name', get_pypi_name_wrapper)
+    def test_add_setup_py_requires_single_variable(self):
+        """
+        Test add_setup_py_requires with a single non-literal object
+        """
+        open_name = 'buildreq.util.open_auto'
+        content = "install_requires='reqname'"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            self.reqs.add_setup_py_requires('filename', [])
+
+        self.assertEqual(self.reqs.buildreqs, set(['pypi(reqname)']))
+        self.assertEqual(self.reqs.requires['python3'], set(['pypi(reqname)']))
+
+    def test_scan_for_configure_setup(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'setup.py'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-distutils3']))
+
+    def test_scan_for_configure_cmake(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'CMakeLists.txt'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-cmake']))
+
+    def test_scan_for_configure_scons(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'SConstruct'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-scons']))
+
+    def test_scan_for_configure_meson(self):
+        """
+        Test scan_for_configure with a mocked package structure. There is so
+        much to test here that uses the same logic, a representative test
+        should be sufficient.
+        """
+        conf = config.Config("")
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'meson.build'), 'w').close()
+
+            self.reqs.scan_for_configure(tmpd, "", conf)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['buildreq-meson']))
+
+    def test_scan_for_configure_pypi(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test pypi metadata handling.
+        """
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        orig_pypi_name = buildreq.pypidata.get_pypi_name
+        orig_pypi_meta = buildreq.pypidata.get_pypi_metadata
+        name = "name"
+        requires = ["abc",
+                    "def"]
+        pypi_requires = set(f"pypi({x})" for x in requires)
+        summary = "summary"
+        content = json.dumps({"name": name,
+                              "summary": summary,
+                              "requires": requires})
+        buildreq.pypidata.get_pypi_name = MagicMock(return_value=True)
+        buildreq.pypidata.get_pypi_metadata = MagicMock(return_value=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
+
+        ssummary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+        buildreq.pypidata.get_pypi_name = orig_pypi_name
+        buildreq.pypidata.get_pypi_metadata = orig_pypi_meta
+
+        self.assertEqual(self.reqs.pypi_provides, name)
+        self.assertEqual(self.reqs.requires['python3'], pypi_requires)
+        self.assertEqual(ssummary, summary)
+
+    def test_scan_for_configure_pypi_override(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test pypi metadata file override handling.
+        """
+        open_name = 'buildreq.open'
+        orig_summary = buildreq.specdescription.default_summary
+        orig_sscore = buildreq.specdescription.default_summary_score
+        name = "name"
+        summary = "summary"
+        requires = ["req"]
+        pypi_requires = set(f"pypi({x})" for x in requires)
+        content = json.dumps({"name": name,
+                              "summary": summary,
+                              "requires": requires})
+        m_open = mock_open(read_data=content)
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'pyproject.toml'), 'w').close()
+            open(os.path.join(tmpd, 'pypi.json'), 'w').close()
+            with patch(open_name, m_open, create=True):
+                self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
+
+        ssummary = buildreq.specdescription.default_summary
+        buildreq.specdescription.default_summary = orig_summary
+        buildreq.specdescription.default_summary_score = orig_sscore
+
+        self.assertEqual(self.reqs.pypi_provides, name)
+        self.assertEqual(self.reqs.requires['python3'], pypi_requires)
+        self.assertEqual(ssummary, summary)
+
+    def test_scan_for_configure_setup_with_requires(self):
+        """
+        Test scan_for_configure when distutils is being used for the build
+        pattern to test requires.txt handling.
+        """
+        self.reqs.add_pyproject_requires = MagicMock()
+        self.reqs.add_setup_py_requires = MagicMock()
+        self.reqs.grab_python_requirements = MagicMock()
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            os.mkdir(os.path.join(tmpd, 'subdir'))
+            open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close()
+            open(os.path.join(tmpd, 'subdir', 'requires.txt'), 'w').close()
+            self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", conf)
+
+        self.reqs.add_pyproject_requires.assert_not_called()
+        self.reqs.add_setup_py_requires.assert_called_once()
+        self.reqs.grab_python_requirements.assert_called_once()
+
+    def test_parse_cmake_pkg_check_modules(self):
+        """
+        Test parse_cmake to ensure accurate detection of versioned and
+        unversioned pkgconfig modules.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = 'pkg_check_modules(GLIB gio-unix-2.0>=2.46.0 glib-2.0 REQUIRED)'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)']))
+
+    def test_parse_cmake_pkg_check_modules_whitespace(self):
+        """
+        Test parse_cmake to ensure accurate handling of versioned
+        pkgconfig modules with whitespace.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = 'pkg_check_modules(GLIB gio-unix-2.0 >= 2.46.0 glib-2.0 REQUIRED)'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)']))
+
+    def test_parse_cmake_pkg_check_modules_in_a_comment(self):
+        """
+        Test parse_cmake to ensure it ignores pkg_check_modules in comments.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = '''
+# For example, consider the following patch to some CMakeLists.txt.
+#     - pkg_check_modules(FOO REQUIRED foo>=1.0)
+#     + pkg_check_modules(FOO REQUIRED foo>=2.0)
+'''
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set([]))
+
+    def test_parse_cmake_pkg_check_modules_variables(self):
+        """
+        Test parse_cmake to ensure accurate handling of versioned
+        pkgconfig modules with variable version strings.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        content = 'pkg_check_modules(AVCODEC libavcodec${_avcodec_ver} libavutil$_avutil_ver)'
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)']))
+
+    def test_parse_cmake_find_package(self):
+        """
+        Test parse_cmake to ensure accurate handling of find_package.
+        """
+        cmake_modules = {
+            "valid": "valid",
+            "valid_but_commented": "valid_but_commented",
+            "different_name": "another_name",
+            "qt6.module1": "qt6module1",
+            "qt6.module2": "qt6module2",
+            "kf6.module3": "kf6module3",
+            "kf6.module4": "kf6module4",
+            ".module5": "namodule5",
+            ".module6": "namodule6"
+        }
+        content = '''
+find_package(valid)
+#find_package(foo)
+  # find_package(valid_but_commented)
+find_package(different_name)
+find_package(Qt6 stuff
+module1
+module2)
+find_package(KF6 stuff
+module3
+module4
+)
+find_package(NOT_HANDLED_NAMESPACE stuff
+module5
+
+module6
+)
+'''
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'fname'), 'w') as f:
+                f.write(content)
+            self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False)
+
+        self.assertEqual(self.reqs.buildreqs,
+                         set(['valid',
+                              'another_name',
+                              'qt6module1',
+                              'qt6module2',
+                              'kf6module3',
+                              'kf6module4',
+                              'namodule5',
+                              'namodule6']))
+
+    def test_r_desc_field_begin(self):
+        """Test parsing of the first R description field."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, ["foo"])
+
+    def test_r_desc_field_middle(self):
+        """Test parsing of an R description field with surrounding fields."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, ["bar"])
+
+    def test_r_desc_field_end(self):
+        """Test parsing of the last R description field."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field3", "\n".join(lines))
+        self.assertEqual(result, ["baz"])
+
+    def test_r_desc_field_middle_multiple_lines(self):
+        """Test parsing of a multi-line R description field, with surrounding fields."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar1, bar2,",
+            "  bar3, bar4",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, ["bar1", "bar2", "bar3", "bar4"])
+
+    def test_r_desc_field_end_multiple_lines(self):
+        """Test parsing of the last R description field, consisting of multiple lines."""
+        lines = [
+            "Field1: foo",
+            "Field2: bar",
+            "Field3: baz1,",
+            "        baz2",
+        ]
+        result = buildreq._get_desc_field("Field3", "\n".join(lines))
+        self.assertEqual(result, ["baz1", "baz2"])
+
+    def test_r_desc_field_middle_one_per_line(self):
+        """Test parsing of an R description field with one entry per line."""
+        lines = [
+            "Field1: foo",
+            "Field2:",
+            "bar1,",
+            "bar2",
+            "Field3: baz",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, ["bar1", "bar2"])
+
+    def test_r_desc_field_trailing_whitespace(self):
+        """Test parsing of an R description field with trailing whitespace."""
+        lines = [
+            "Field1: foo1, foo2    ",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, ["foo1", "foo2"])
+
+    def test_r_desc_field_trailing_comma(self):
+        """Test parsing of an R description field with trailing comma."""
+        lines = [
+            "Field1: foo1, foo2,",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, ["foo1", "foo2"])
+
+    def test_r_desc_field_empty(self):
+        """Test parsing of an R description field with an empty value."""
+        lines = [
+            "Field1:",
+        ]
+        result = buildreq._get_desc_field("Field1", "\n".join(lines))
+        self.assertEqual(result, [])
+
+    def test_r_desc_field_missing(self):
+        """Test parsing of an R description field that is missing."""
+        lines = [
+            "Field1: foo, bar",
+        ]
+        result = buildreq._get_desc_field("Field2", "\n".join(lines))
+        self.assertEqual(result, [])
+
+    def test_parse_r_desc_depends(self):
+        """Test parsing of R description Depends field."""
+        pkgs = ['R-pkg1']
+        open_name = 'buildreq.util.open_auto'
+        content = 'Depends: pkg1'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg1' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_imports(self):
+        """Test parsing of an R description Imports field."""
+        pkgs = ['R-pkg2']
+        open_name = 'buildreq.util.open_auto'
+        content = 'Imports: pkg2'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg2' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_linkingto(self):
+        """Test parsing of an R description LinkingTo field."""
+        pkgs = ['R-pkg3']
+        open_name = 'buildreq.util.open_auto'
+        content = 'LinkingTo: pkg3'
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg3' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_multiple(self):
+        """Test parsing of an R description file that captures multiple fields."""
+        pkgs = [
+            'R-pkg1',
+            'R-pkg2',
+            'R-pkg3',
+            'R-pkg4',
+        ]
+        open_name = 'buildreq.util.open_auto'
+        content = [
+            'Field1: foo',
+            'Imports: pkg1, pkg2,',
+            '            pkg3   ',
+            'LinkingTo: pkg4',
+            'FieldFoo: bar',
+        ]
+        m_open = mock_open(read_data='\n'.join(content))
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertFalse('R-foo' in self.reqs.buildreqs)
+        self.assertFalse('R-bar' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg1' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg2' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg3' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg4' in self.reqs.buildreqs)
+
+    def test_parse_r_desc_not_in_os(self):
+        """Test parsing of an R description file with some non-OS packages."""
+        pkgs = [
+            'R-pkg1',
+        ]
+        open_name = 'buildreq.util.open_auto'
+        content = [
+            'Imports: pkg1, pkg2',
+            'Depends: pkg3',
+        ]
+        m_open = mock_open(read_data='\n'.join(content))
+        with patch(open_name, m_open):
+            self.reqs.parse_r_description('filename', pkgs)
+        self.assertTrue('R-pkg1' in self.reqs.buildreqs)
+        self.assertTrue('R-pkg1' in self.reqs.requires[None])
+        # Names absent from the os-packages list are also added, because the
+        # DESCRIPTION file is considered authoritative.
+        for pkg in ['R-pkg2', 'R-pkg3']:
+            self.assertTrue(pkg in self.reqs.buildreqs)
+            self.assertTrue(pkg in self.reqs.requires[None])
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_check.py

@@ -0,0 +1,215 @@
+import os
+import tempfile
+import unittest
+from unittest.mock import mock_open, patch
+
+import buildreq
+import check
+import config
+import tarball
+
+def mock_generator(rv=None):
+    def mock_f(*args, **kwargs):
+        return rv
+
+    return mock_f
+
+
+class TestTest(unittest.TestCase):
+    backup_isfile = check.os.path.isfile
+
+    @classmethod
+    def setUpClass(self):
+        self.open_name = 'check.util.open_auto'
+        check.os.path.isfile = mock_generator(True)
+
+    @classmethod
+    def tearDownClass(self):
+        check.os.path.isfile = self.backup_isfile
+
+    def setUp(self):
+        check.tests_config = ''
+
+    def test_check_regression(self):
+        """
+        Test check_regression
+        """
+        def mock_parse_log(log):
+            return ',120,100,20,0,0'
+
+        parse_log_backup = check.count.parse_log
+        check.count.parse_log = mock_parse_log
+        m_open = mock_open()
+        open_name = 'util.open'
+        with patch(open_name, m_open, create=True):
+            check.check_regression('pkgdir', False, -1)
+
+        check.count.parse_log = parse_log_backup
+
+        exp_call = unittest.mock.call().write('Total : 120\n'
+                                              'Pass : 100\n'
+                                              'Fail : 20\n'
+                                              'Skip : 0\n'
+                                              'XFail : 0\n')
+        self.assertIn(exp_call, m_open.mock_calls)
+
+    def test_check_regression_multi(self):
+        """
+        Test check_regression with multiple results
+        """
+        def mock_parse_log(log):
+            return 'test-a,120,100,20,0,0\ntest-b,10,5,3,2,1'
+
+        parse_log_backup = check.count.parse_log
+        check.count.parse_log = mock_parse_log
+        m_open = mock_open()
+        open_name = 'util.open'
+        with patch(open_name, m_open, create=True):
+            check.check_regression('pkgdir', False, -1)
+
+        check.count.parse_log = parse_log_backup
+
+        exp_call = unittest.mock.call().write('Package : test-a\n'
+                                              'Total : 120\n'
+                                              'Pass : 100\n'
+                                              'Fail : 20\n'
+                                              'Skip : 0\n'
+                                              'XFail : 0\n'
+                                              'Package : test-b\n'
+                                              'Total : 10\n'
+                                              'Pass : 5\n'
+                                              'Fail : 3\n'
+                                              'Skip : 2\n'
+                                              'XFail : 1\n')
+        self.assertIn(exp_call, m_open.mock_calls)
+
+    def test_scan_for_tests_makecheck_in(self):
+        """
+        Test scan_for_tests with makecheck suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.in'])
+        content = 'check:'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "configure"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'make %{?_smp_mflags} check')
+
+    def test_scan_for_tests_makecheck_am(self):
+        """
+        Test scan_for_tests with makecheck suite via Makefile.am
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.am'])
+        m_open = mock_open()
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "configure_ac"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'make %{?_smp_mflags} check')
+
+    def test_scan_for_tests_perlcheck_PL(self):
+        """
+        Test scan_for_tests with perlcheck suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.PL'])
+        conf.default_pattern = "cpan"
+        check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test')
+
+    def test_scan_for_tests_perlcheck_in(self):
+        """
+        Test scan_for_tests with perlcheck suite via Makefile.in
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['Makefile.in'])
+        content = 'test:'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "cpan"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test')
+
+    def test_scan_for_tests_setup(self):
+        """
+        Test scan_for_tests with setup.py suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['setup.py'])
+        content = 'test_suite'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "distutils3"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'PYTHONPATH=%{buildroot}$(python -c "import sys; print(sys.path[-1])") '
+                         'python setup.py test')
+
+    def test_scan_for_tests_cmake(self):
+        """
+        Test scan_for_tests with cmake suite
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['CMakeLists.txt'])
+        content = 'enable_testing'
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            conf.default_pattern = "cmake"
+            check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+
+        check.os.listdir = listdir_backup
+        self.assertEqual(check.tests_config,
+                         'cd clr-build; make test')
+
+    def test_scan_for_tests_tox_requires(self):
+        """
+        Test scan_for_tests with tox.ini in the files list, should add several
+        build requirements
+        """
+        reqs = buildreq.Requirements("")
+        conf = config.Config("")
+        tcontent = tarball.Content("", "", "", [], conf, "")
+        listdir_backup = os.listdir
+        check.os.listdir = mock_generator(['tox.ini'])
+        check.scan_for_tests('pkgdir', conf, reqs, tcontent)
+        check.os.listdir = listdir_backup
+        self.assertEqual(reqs.buildreqs,
+                         set(['pypi-tox',
+                              'pypi-pytest',
+                              'pypi-virtualenv',
+                              'pypi-pluggy',
+                              'pypi(py)']))
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=True)

tests/test_commitmessage.py

@@ -0,0 +1,245 @@
+import unittest
+import unittest.mock as mock
+import os
+import tempfile
+import build
+import commitmessage
+import config
+import tarball
+
+
+class TestCommitmessage(unittest.TestCase):
+
+    def setUp(self):
+        self.workingdir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.workingdir.cleanup()
+
+    def test_is_header(self):
+        """
+        Test is_header function with list of lines. First and last line, line
+        followed by a line containing '---', and lines after a blank '' line
+        should be recognized as headers. Last line recognized as header because
+        it is a relevant ending point.
+        """
+        lines = ['line1',  # True
+                 'line2',  # False
+                 '',       # False
+                 'line4',  # True
+                 'line5',  # True
+                 '---',    # False
+                 'line7',  # False
+                 'line8']  # True
+        for idx, line in enumerate(lines):
+            print(line, idx)
+            if idx in [0, 3, 4, 7]:
+                self.assertTrue(commitmessage.is_header(lines, idx))
+            else:
+                self.assertFalse(commitmessage.is_header(lines, idx))
+
+    def test_find_in_line(self):
+        """
+        Trivially tests commitmessage.find_in_line(). Just makes sure results
+        evaluate to the correct bool.
+        """
+        self.assertTrue(commitmessage.find_in_line(r'Version', 'AVersionInThisLine'))
+        self.assertFalse(commitmessage.find_in_line(r'z', 'the quick brown fox jumps over the lady dog'))
+
+    def test_process_NEWS(self):
+        """
+        Test process_NEWS() function with valid newsfile provided
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                newsfile.write(GOOD_NEWS)
+            # commitmessage returned will have an empty string as first and
+            # last items
+            expected_msg = [""] + GOOD_NEWS.split('\n')[3:13]
+            expected_cvs = set()
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd),
+                             (expected_msg, expected_cvs))
+
+    def test_process_NEWS_bad_news(self):
+        """
+        Test process_NEWS() function with irrelevant newsfile provided
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                # make GOOD_NEWS irrelevant by replacing current version
+                newsfile.write(GOOD_NEWS.replace('0.0.1', '0.0.0'))
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd), ([], set()))
+
+    def test_process_NEWS_good_cves(self):
+        """
+        Test process_NEWS() function with valid newsfile and CVEs
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                # give GOOD_NEWS some CVEs
+                newsfile.write(GOOD_NEWS.replace('change2.1', 'CVE-2-1')
+                                        .replace('change2.2', 'CVE-2-2'))
+            # commitmessage returned will have an empty string as first and
+            # last items.
+            # replace change2.* strings with CVE strings
+            expected_msg = [""] + GOOD_NEWS.replace('change2.1', 'CVE-2-1')\
+                                           .replace('change2.2', 'CVE-2-2')\
+                                           .split('\n')[3:13]
+            expected_cvs = set(['CVE-2-1', 'CVE-2-2'])
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd),
+                             (expected_msg, expected_cvs))
+
+    def test_process_NEWS_long(self):
+        """
+        Test process_NEWS() function with valid newsfile provided, but relevant
+        block is longer than 15 lines, causing it to be truncated.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            long_news = GOOD_NEWS.replace('text explaining change2.2',
+                                          '1\n2\n3\n4\n5\n6\n7\n8\n9\n')
+            with open(os.path.join(tmpd, 'NEWS'), 'w') as newsfile:
+                newsfile.write(long_news)
+            # commitmessage returned will have an empty string as first and
+            # last items, extend the expected message with extra lines and
+            # truncate message.
+            expected_msg = [""] + GOOD_NEWS.split('\n')[3:11]
+            expected_msg.extend(['1', '2', '3', '4', '5', '6', '7', '',
+                                 '(NEWS truncated at 15 lines)', ''])
+            expected_cvs = set()
+            self.assertEqual(commitmessage.process_NEWS('NEWS', '0.0.0', '', '0.0.1', tmpd),
+                             (expected_msg, expected_cvs))
+
+    def test_guess_commit_message(self):
+        """
+        Test guess_commit_message() with mocked internal functions and both
+        commitmessage information and cves available from newsfile.
+        """
+        conf = config.Config("")
+        conf.old_version = "0.0.0"
+        tcontent = tarball.Content("", "testball", "0.0.1", [], conf, self.workingdir.name)
+        conf.content = tcontent
+        process_NEWS_backup = commitmessage.process_NEWS
+
+        def mock_process_NEWS(newsfile, old_version, name, version, download_path):
+            return (['', 'commit', 'message', 'with', 'cves', ''],
+                    set(['cve1', 'cve2']))
+
+        commitmessage.process_NEWS = mock_process_NEWS
+        open_name = 'util.open_auto'
+        with mock.patch(open_name, create=True) as mock_open:
+            mock_open.return_value = mock.MagicMock()
+            conf.rewrite_config_opts = mock.Mock()
+            commitmessage.guess_commit_message("", conf, tcontent)
+            # reset mocks before asserting so a failure doesn't cascade to
+            # other tests
+            commitmessage.process_NEWS = process_NEWS_backup
+            fh = mock_open.return_value.__enter__.return_value
+            fh.write.assert_called_with(
+                'testball: Autospec creation for update from version 0.0.0 to '
+                'version 0.0.1\n\n\ncommit\nmessage\nwith\ncves\n\n\ncommit\n'
+                'message\nwith\ncves\n\nCVEs fixed in this build:\ncve1\ncve2'
+                '\n\n')
+
+    def test_guess_commit_message_cve_config(self):
+        """
+        Test guess_commit_message() with mocked internal functions and both
+        commitmessage information and cves available from newsfile. A cve is
+        also available from config, which changes the first line of the commmit
+        message.
+        """
+        conf = config.Config("")
+        tcontent = tarball.Content("", "testball", "0.0.1", [], conf, self.workingdir.name)
+        conf.content = tcontent
+        process_NEWS_backup = commitmessage.process_NEWS
+
+        def mock_process_NEWS(newsfile, old_version, name, version, download_path):
+            return (['', 'commit', 'message', 'with', 'cves', ''],
+                    set(['cve1', 'cve2']))
+
+        commitmessage.process_NEWS = mock_process_NEWS
+        conf.cves = set(['CVE-1234-5678'])
+        conf.old_version = None  # Allow cve title to be set
+        open_name = 'util.open_auto'
+        with mock.patch(open_name, create=True) as mock_open:
+            mock_open.return_value = mock.MagicMock()
+            conf.rewrite_config_opts = mock.Mock()
+            commitmessage.guess_commit_message("", conf, tcontent)
+            # reset mocks before asserting so a failure doesn't cascade to
+            # other tests
+            commitmessage.process_NEWS = process_NEWS_backup
+            fh = mock_open.return_value.__enter__.return_value
+            fh.write.assert_called_with(
+                'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n'
+                'cves\n\n\ncommit\nmessage\nwith\ncves\n\nCVEs fixed in this '
+                'build:\nCVE-1234-5678\ncve1\ncve2\n\n')
+
+    def test_guess_commit_message_imported_key(self):
+        """
+        Test guess_commit_message() with mocked internal functions and both
+        commitmessage information and cves available from newsfile. A cve is
+        also available from config, which changes the first line of the commmit
+        message. Additionally there is imported key info that will be displayed
+        at the end of the message.
+        """
+        conf = config.Config("")
+        tcontent = tarball.Content("", "testball", "0.0.1", [], conf, self.workingdir.name)
+        conf.content = tcontent
+        process_NEWS_backup = commitmessage.process_NEWS
+
+        def mock_process_NEWS(newsfile, old_version, name, version, download_path):
+            return (['', 'commit', 'message', 'with', 'cves', ''],
+                    set(['cve1', 'cve2']))
+
+        commitmessage.process_NEWS = mock_process_NEWS
+        conf.cves = set(['CVE-1234-5678'])
+        conf.old_version = None  # Allow cve title to be set
+        open_name = 'util.open_auto'
+        with mock.patch(open_name, create=True) as mock_open:
+            mock_open.return_value = mock.MagicMock()
+            conf.rewrite_config_opts = mock.Mock()
+            commitmessage.guess_commit_message("keyinfo content", conf, tcontent)
+            # reset mocks before asserting so a failure doesn't cascade to
+            # other tests
+            commitmessage.process_NEWS = process_NEWS_backup
+            fh = mock_open.return_value.__enter__.return_value
+            fh.write.assert_called_with(
+                'testball: Fix for CVE-1234-5678\n\n\ncommit\nmessage\nwith\n'
+                'cves\n\n\ncommit\nmessage\nwith\ncves\n\nCVEs fixed in this '
+                'build:\nCVE-1234-5678\ncve1\ncve2\n\nKey imported:\nkeyinfo '
+                'content\n')
+
+    def test_scan_for_changes(self):
+        """
+        Tests scan_for_changes using temporary directories
+        """
+        conf = config.Config("")
+        with tempfile.TemporaryDirectory() as tmpd:
+            with open(os.path.join(tmpd, 'changelog.txt'), 'w') as newsfile:
+                newsfile.write('new changelog file')
+
+            with tempfile.TemporaryDirectory() as tmpd1:
+                commitmessage.scan_for_changes(tmpd1, tmpd, conf.transforms)
+                self.assertTrue(os.path.isfile(tmpd1 + '/ChangeLog'))
+
+
+GOOD_NEWS = """
+GOOD NEWS -- History of user-visible changes.
+
+* Version 0.0.1
+
+change1.1
+change1.2
+
+change2.1
+text explaining change2.1
+change2.2
+text explaining change2.2
+
+* Version 0.0.0
+
+This better not show up
+"""
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_config.py

@@ -0,0 +1,89 @@
+import unittest
+import config
+
+
+# Input for tarball.detect_build_from_url method tests
+# Structure: (url, build_pattern)
+BUILD_PAT_URL = [
+    ("https://cran.r-project.org/src/contrib/raster_3.0-12.tar.gz", "R"),
+    ("http://pypi.debian.net/argparse/argparse-1.4.0.tar.gz", "distutils3"),
+    ("https://pypi.python.org/packages/source/T/Tempita/Tempita-0.5.2.tar.gz", "distutils3"),
+    ("https://cpan.metacpan.org/authors/id/T/TO/TODDR/IO-Tty-1.14.tar.gz", "cpan"),
+    ("http://search.cpan.org/CPAN/authors/id/D/DS/DSKOLL/IO-stringy-2.111.tar.gz", "cpan"),
+    ("https://pecl.php.net//get/lua-2.0.6.tgz", "phpize"),
+]
+
+
+def detect_build_test_generator(url, build_pattern):
+    """Create test for tarball.detect_build_from_url method."""
+    def generator(self):
+        """Test template."""
+        conf = config.Config("")
+        conf.detect_build_from_url(url)
+        self.assertEqual(build_pattern, conf.default_pattern)
+
+    return generator
+
+
+def create_dynamic_tests():
+    # Create tests for config.detect_build_from_url method.
+    for url, build_pattern in BUILD_PAT_URL:
+        test_name = 'test_pat_{}'.format(url)
+        test = detect_build_test_generator(url, build_pattern)
+        setattr(TestConfig, test_name, test)
+
+
+class TestConfig(unittest.TestCase):
+
+    def test_set_build_pattern(self):
+        """
+        Test set_build_pattern with sufficient pattern strength
+        """
+        conf = config.Config("")
+        conf.set_build_pattern("configure_ac", 1)
+        self.assertEqual(conf.default_pattern, "configure_ac")
+        self.assertEqual(conf.pattern_strength, 1)
+
+    def test_set_build_pattern_low_strength(self):
+        """
+        Test set_build_pattern with low pattern strength, nothing in the module
+        should change
+        """
+        conf = config.Config("")
+        conf.pattern_strength = 2
+        conf.set_build_pattern("configure_ac", 1)
+        self.assertEqual(conf.default_pattern, "make")
+        self.assertEqual(conf.pattern_strength, 2)
+
+    def test_validate_extras_content_bad_glob(self):
+        """
+        Test validate_extras_content with more than one glob in a directory
+        """
+        conf = config.Config("")
+        lines = ['/bad*path*/file']
+        new_lines = conf.validate_extras_content(lines, 'bad_glob')
+        self.assertEqual(len(new_lines), 0)
+
+    def test_validate_extras_content_good_single_glob(self):
+        """
+        Test validate_extras_content with a single glob in a directory
+        """
+        conf = config.Config("")
+        lines = ['/good*/file']
+        new_lines = conf.validate_extras_content(lines, 'good_single_glob')
+        self.assertEqual(new_lines, [lines[0].split('/')])
+
+    def test_validate_extras_content_good_multi_glob(self):
+        """
+        Test validate_extras_content with a multiple valid globs
+        """
+        conf = config.Config("")
+        lines = ['/path1', '/good*/glob*/file', '/path3']
+        new_lines = conf.validate_extras_content(lines, 'good_multi_glob')
+        self.assertEqual(new_lines, ['/path1', lines[1].split('/'), '/path3'])
+
+# Create dynamic tests
+create_dynamic_tests()
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_count.py

@@ -0,0 +1,555 @@
+import unittest
+from unittest.mock import mock_open, patch
+import count
+
+pats = [
+    # acl
+    ('[22] $ rm -Rf d -- ok-',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('17 commands (17 passed, 1 failed)-',
+     [18, 17, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # alembic
+    ('Ran 678 tests in 5.175s',
+     [678, 678, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('OK (SKIP=15)',
+     [15, 0, 0, 0, 15, 0, 0, 0, 0, 0]),
+    ('OK (skipped=16)',
+     [16, 0, 0, 0, 16, 0, 0, 0, 0, 0]),
+    # anyjson
+    ('test_implementations.test_default_serialization ... ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('test_implementations.test_default_serialization ... skipped',
+     [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]),
+    # apr
+    ('testatomic          :  SUCCESS',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # cryptography
+    ('================= 76230 passed, 267 skipped in 140.23 seconds ==================',
+     [76497, 76230, 0, 0, 267, 0, 0, 0, 0, 0]),
+    ('================== 47 passed, 2 error in 10.36 seconds =========================',
+     [49, 47, 2, 0, 0, 0, 0, 0, 0, 0]),
+    ('================ 10 failed, 16 passed, 4 error in 0.16 seconds =================',
+     [30, 16, 14, 0, 0, 0, 0, 0, 0, 0]),
+    ('========================== 43 passed in 2.90 seconds ===========================',
+     [43, 43, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('========================== 43 passed in 2.90s ===========================',
+     [43, 43, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('======= 28 failed, 281 passed, 13 skipped, 10 warnings in 28.48 seconds ========',
+     [332, 281, 38, 0, 13, 0, 0, 0, 0, 0]),
+    ('===================== 5 failed, 318 passed in 1.06 seconds =====================',
+     [323, 318, 5, 0, 0, 0, 0, 0, 0, 0]),
+    ('===================== 5 failed, 9 passed, 7 xfailed in 1.06 seconds ============',
+     [21, 9, 5, 7, 0, 0, 0, 0, 0, 0]),
+    ('============= 1628 passed, 72 skipped, 4 xfailed in 146.26 seconds =============',
+     [1704, 1628, 0, 4, 72, 0, 0, 0, 0, 0]),
+    ('=============== 119 passed, 2 skipped, 54 error in 2.19 seconds ================',
+     [175, 119, 54, 0, 2, 0, 0, 0, 0, 0]),
+    ('========== 1 failed, 74 passed, 10 skipped, 55 error in 2.05 seconds ===========',
+     [140, 74, 56, 0, 10, 0, 0, 0, 0, 0]),
+    ('==================== 68 passed, 1 warnings in 0.12 seconds =====================',
+     [69, 68, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('================ 3 failed, 250 passed, 3 error in 3.28 seconds =================',
+     [256, 250, 6, 0, 0, 0, 0, 0, 0, 0]),
+    ('=============== 1 failed, 407 passed, 10 skipped in 4.71 seconds ===============',
+     [418, 407, 1, 0, 10, 0, 0, 0, 0, 0]),
+    ('=============== 1 failed, 407 passed, 10 skipped in 4.71s ===============',
+     [418, 407, 1, 0, 10, 0, 0, 0, 0, 0]),
+    ('========================== 1 skipped in 0.79 seconds ===========================',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('========================== 1 skipped in 0.79s ===========================',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('=========================== 3 error in 0.41 seconds ============================',
+     [3, 0, 3, 0, 0, 0, 0, 0, 0, 0]),
+    ('=========================== 3 error in 0.41s ============================',
+     [3, 0, 3, 0, 0, 0, 0, 0, 0, 0]),
+    ('================= 68 passed, 1 pytest-warnings in 0.09 seconds =================',
+     [68, 68, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('================= 68 passed, 1 pytest-warnings in 0.09s =================',
+     [68, 68, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('===== 21 failed, 73 passed, 5 skipped, 2 pytest-warnings in 34.81 seconds ======',
+     [99, 73, 21, 0, 5, 0, 0, 0, 0, 0]),
+    ('===== 21 failed, 73 passed, 5 skipped, 2 pytest-warnings in 34.81s ======',
+     [99, 73, 21, 0, 5, 0, 0, 0, 0, 0]),
+    # swift
+    ('========= 1 failed, 1287 passed, 1 warnings, 62 error in 35.77 seconds =========',
+     [1351, 1287, 64, 0, 0, 0, 0, 0, 0, 0]),
+    ('===== 487 failed, 4114 passed, 32 skipped, 1 pytest-warnings, 34 error in 222.82 seconds =====',
+     [4667, 4114, 521, 0, 32, 0, 0, 0, 0, 0]),
+    # tox
+    ('======== 199 passed, 38 skipped, 1 xpassed, 1 warnings in 5.76 seconds =========',
+     [239, 200, 1, 0, 38, 0, 0, 0, 0, 0]),
+    # augeas
+    ('# TOTAL: 215\n'
+     '# PASS:  212\n'
+     '# SKIP:  3\n'
+     '# XFAIL: 0\n'
+     '# FAIL:  0\n'
+     '# XPASS: 0\n'
+     '# ERROR: 0',
+     [215, 212, 0, 0, 3, 0, 0, 0, 0, 0]),
+    # autoconf
+    ('493 tests behaved as expected\n'
+     '10 tests were skipped.\n'
+     '495: AC_FUNC_STRNLEN                                 ok\n'
+     '344: Erlang                                          skipped (erlang.at:30)\n'
+     '26: autoupdating macros recursively                 expected failure (tools.at:945)',
+    [503, 493, 0, 0, 10, 3, 1, 0, 1, 1]),
+    # bison
+    ('470 tests were successful',
+    [470, 470, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # binutils
+    ('# of expected passes\t1144\n'
+     '# of expected failures\t57\n'
+     '# of untested testcases\t1\n'
+     '# of unsupported tests\t12\n'
+     '# of unexpected failures\t1\n',
+     [1214, 1144, 1, 57, 12, 0, 0, 0, 0, 0]),
+    # ccache
+    ('PASSED: 448 assertions, 88 tests, 10 suites',
+     [88, 88, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # rubygem-rack
+    ('701 tests, 2292 assertions, 0 failures, 0 errors',
+     [701, 701, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # curl
+    ('TESTDONE: 680 tests out of 686 reported OK: 99%',
+     [686, 680, 6, 0, 0, 0, 0, 0, 0, 0]),
+    # gcc
+    ('All 4 tests passed\n'
+     'PASS: test-strtol-16\n'
+     'FAIL: test-strtol-32',
+     [4, 4, 0, 0, 0, 2, 1, 1, 0, 0]),
+    # gdbm
+    ('All 22 tests were successful.',
+     [22, 22, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # glibc
+    ('3 FAIL\n'
+     '2182 PASS\n'
+     '1 UNRESOLVED\n'
+     '199 XFAIL\n'
+     '3 XPASS',
+     [2387, 2185, 3, 199, 0, 0, 0, 0, 0, 0]),
+    # libxml2
+    ('Total 2908 tests, no errors',
+     [2908, 2908, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('Total: 1171 functions, 291083 tests, 0 errors',
+     [1171, 1171, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # zlib
+    ('*** zlib shared test OK ***',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # e2fsprogs
+    ('153 tests succeeded     1 tests failed',
+     [154, 153, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # expect
+    ('all.tcl:     Total   41     Passed   29     Skipped   2     Failed   10',
+     [41, 29, 10, 0, 2, 0, 0, 0, 0, 0]),
+    # expat
+    ('50%: Checks: 50, Failed: 25',
+     [50, 25, 25, 0, 0, 0, 0, 0, 0, 0]),
+    # flex
+    ('Tests succeeded: 47\n'
+     'Tests FAILED: 3',
+     [50, 47, 3, 0, 0, 0, 0, 0, 0, 0]),
+    # TAP and perl-Capture-tiny
+    ('ok 580 - tee_merged|sys|stderr|short = got STDERR',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('not ok 580 - tee_merged|sys|stderr|short = got STDERR',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    ('not ok 580 - tee_merged|sys|stderr|short = got STDERR  # TODO known breakage',
+     [0, 0, 0, 0, 0, 0, 0, 0, 1, 0]),
+    ('ok 9',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('not ok 9',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # tcpdump
+    ('    1 test failed\n'
+     ' 154 tests passed',
+     [155, 154, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('    2 tests failed\n'
+     ' 154 tests passed',
+     [156, 154, 2, 0, 0, 0, 0, 0, 0, 0]),
+    # R packages
+    ('* checking top-level files ... OK',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('* checking top-level files ... PASSED.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('* checking top-level files ... SKIPPED',
+     [0, 0, 0, 0, 0, 0, 0, 0, 0, 1]),
+    # python
+    ('365 tests OK.\n'
+     '22 tests skipped:',
+     [387, 365, 0, 0, 22, 0, 0, 0, 0, 0]),
+    # jemalloc
+    ('Test suite summary: pass: 30/33, skip: 3/33, fail: 0/33',
+     [33, 30, 0, 0, 3, 0, 0, 0, 0, 0]),
+    # util-linux
+    ('  All 160 tests PASSED',
+     [160, 160, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # onig
+    ("OK: // 'a'",
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # nss
+    ('cert.sh: #101: Import chain-2-serverCA-ec CA -t u,u,u for localhost.localdomain (ext.)  - PASSED\n'
+     'Passed:             13036\n'
+     'Failed:             6\n'
+     'Failed with core:   0\n'
+     'Unknown status:     0',
+     [13042, 13036, 6, 0, 0, 1, 1, 0, 0, 0]),
+    # nss
+    ('cert.sh: #101: Import chain-2-serverCA-ec CA -t u,u,u for localhost.localdomain (ext.)  - FAILED\n'
+     'Passed:             13036\n'
+     'Failed:             6\n'
+     'Failed with core:   0\n'
+     'Unknown status:     0',
+     [13042, 13036, 6, 0, 0, 0, 0, 1, 0, 0]),
+    # rsync
+    ('     34 passed\n'
+     '     5 skipped',
+     [39, 34, 0, 0, 5, 0, 0, 0, 0, 0]),
+    # mariadb
+    ('50% tests passed, 20 tests failed out of 40',
+     [40, 20, 20, 0, 0, 0, 0, 0, 0, 0]),
+    # python-runtime-tests
+    ('FAILED (KNOWNFAIL=6, SKIP=18, errors=6)',
+     [30, 0, 6, 6, 18, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1, errors=499, skipped=48)',
+     [548, 0, 1, 499, 48, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1, errors=499)',
+     [500, 0, 1, 499, 0, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1)',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('FAILED (errors=1)',
+     [1, 0, 0, 1, 0, 0, 0, 0, 0, 0]),
+    ('OK (KNOWNFAIL=5, SKIP=15)',
+     [20, 0, 0, 5, 15, 0, 0, 0, 0, 0]),
+    # qpid-python
+    ('Totals: 318 tests, 200 passed, 112 skipped, 0 ignored, 6 failed',
+     [318, 200, 6, 0, 112, 0, 0, 0, 0, 0]),
+    # PyYAML
+    ('TESTS: 2577',
+     [2577, 2577, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # sudo
+    ('visudo: 7/7 tests passed; 0/7 tests failed',
+     [7, 7, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('check_symbols: 7 tests run, 0 errors, 100% success rate',
+     [7, 7, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # R
+    ("running code in 'reg-examples1.R' ... OK",
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Status: 1 ERROR, 1 WARNING, 4 NOTEs',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('OK: 749 SKIPPED: 4 FAILED: 2',
+     [755, 749, 2, 0, 4, 0, 0, 0, 0, 0]),
+    # php
+    ('Number of tests : 13526              9794\n'
+     'Tests skipped   : 3732 ( 27.6%) --------\n'
+     'Tests warned    :    0 (  0.0%) (  0.0%)\n'
+     'Tests failed    :   12 (  0.1%) (  0.1%)\n'
+     'Expected fail   :   31 (  0.2%) (  0.3%)\n'
+     'Tests passed    : 9751 ( 72.1%) ( 99.6%)',
+     [13526, 9751, 12, 31, 3732, 0, 0, 0, 0, 0]),
+    # rubygem/rake
+    ('174 runs, 469 assertions, 0 failures, 0 errors, 0 skips',
+     [174, 174, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # cryptsetup
+    (' [OK]',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # lzo
+    (' test passed.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # lsof
+    ('LTnlink ... OK',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('LTnfs ... ERROR!!!',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # libaio
+    ('Pass: 11  Fail: 1',
+     [12, 11, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # gawk
+    ('ALL TESTS PASSED',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # gptfdisk
+    ('**SUCCESS**',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # boost
+    ('**passed** ...',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('8 errors detected.',
+     [8, 0, 8, 0, 0, 0, 0, 0, 0, 0]),
+    ('8 failures detected.',
+     [8, 0, 8, 0, 0, 0, 0, 0, 0, 0]),
+    # make
+    ('534 Tests in 118 Categories Complete ... No Failures',
+     [534, 534, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # icu4c
+    ('---[OK]',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # libxslt
+    ('Pass 1',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # apr-util
+    (':  SUCCESS',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # bash
+    ('< Failed 126 of 1378 Unicode tests',
+     [1378, 1252, 126, 0, 0, 0, 0, 0, 0, 0]),
+    # crudini
+    ('Test 95 OK (line 460)',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Test 95 BAD',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # discount
+    ('Reddit-style automatic links ............................... OK',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Reddit-style automatic links .............................. BAD',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # libjpeg-turbo
+    ('JPEG -> RGP Top-Down 2/1 ... Passed.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # zlib
+    ('*** zlib test OK ***',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('*** zlib 64-bit test OK ***',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # LVM2
+    ('valgrind pool awareness ... fail',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    ('dfa with non-print regex chars ... pass',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # keyring
+    ('==== 76 passed, 62 skipped, 50 xfailed, 14 xpassed, 2 warnings, 32 error in 2.13 seconds ====',
+     [236, 90, 34, 50, 62, 0, 0, 0, 0, 0]),
+    # openblas
+    ('  ----- PASS -----',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('  ----- FAIL -----',
+     [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]),
+    # rubygem-hashie
+    ('545 examples, 0 failures, 1 pending',
+     [546, 545, 0, 0, 1, 0, 0, 0, 0, 0]),
+    # rubygem-warden
+    ('215 examples, 14 failures',
+     [229, 215, 14, 0, 0, 0, 0, 0, 0, 0]),
+    # rubygem-ansi
+    ('Executed 12 tests with 7 passing, 5 errors.',
+     [12, 7, 5, 0, 0, 0, 0, 0, 0, 0]),
+    # vim
+    ('Executed 12 tests',
+     [12, 12, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # rubygem-formatador
+    ('  9 succeeded in 0.00375661 seconds',
+     [9, 9, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # pigz
+    ('./pigz -kf pigz.c ; ./pigz -t pigz.c.gz',
+     [2, 2, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('./pigz -kfb 32 pigz.c',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # netifaces
+    ('Interface lo:',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # btrfs-progs
+    ('    [TEST]    001-bad-file-extent-bytenr',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('    [NOTRUN]  Need to validate root privileges',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('test failed for case',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # chrpath
+    ('success: chrpath changed rpath to larger path.',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('error: chrpath unable to change rpath to larger path.',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('warning: chrpath does not have root permissions',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # yajl
+    ('58/58 tests successful',
+     [58, 58, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # xmlsec1
+    ('    Checking required transforms                            OK',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('    Verify existing signature                             Fail',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('    Checking required transforms                          Skip',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    # xdg-utils
+    ('TOTAL: 4 tests failed, 90 of 116 tests passed. (140 attempted)',
+    [140, 90, 4, 0, 46, 0, 0, 0, 0, 0]),
+    # slang
+    ('Testing argv processing ...Ok',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('./utf8.sl:14:check_sprintf:Test Error',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    # go & golang
+    ('ok    golang.org/x/text/encoding/htmlindex    0.002s',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('--- FAIL: TestParents (0.00s)',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('FAIL	golang.org/x/text/internal	0.002s',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('--- PASS: TestApp_Command (0.00s)',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # valgrind
+    ('== 5 tests, 0 stderr failures, 1 stdout failure, 0 stderrB failures, 0 stdoutB failures, 0 post failures ==',
+     [5, 4, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('== 55 tests, 48 stderr failures, 6 stdout failures, 0 stderrB failures, 0 stdoutB failures, 0 post failures ==',
+     [55, 1, 54, 0, 0, 0, 0, 0, 0, 0]),
+    ('== 125 tests, 12 stderr failures, 0 stdout failures, 0 stderrB failures, 0 stdoutB failures, 0 post failures ==',
+     [125, 113, 12, 0, 0, 0, 0, 0, 0, 0]),
+    # zsh
+    ('46 successful test scripts, 0 failures, 1 skipped',
+     [47, 46, 0, 0, 1, 0, 0, 0, 0, 0]),
+    # glog
+    ('Passed 3 tests',
+     [3, 3, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # hdf5
+    ('Testing h5repack h5repack_szip.h5 -f dset_szip:GZIP=1                  -SKIP-',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('Verifying h5repack h5repack_szip.h5 -f dset_szip:GZIP=1                -SKIP-',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('Verifying h5dump output -f GZIP=1 -m 1024                             *FAILED*',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('Testing h5dump output -f GZIP=1 -m 1024                               *FAILED*',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('Testing h5repack --metadata_block_size=8192                            PASSED',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    ('Verifying h5diff output h5repack_layout.h5 out-meta_long.h5repack_layo PASSED',
+     [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # libconfig
+    ('3 tests; 3 passed, 0 failed',
+     [3, 3, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # libogg
+    ('testing page spill expansion... 0, (0),  granule:0 1, (1),  granule:4103 2, (2),  granule:5127 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing max packet segments... 0, (0),  granule:0 1, (1),  granule:261127 2, (2),  granule:262151 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing very large packets... 0, (0),  granule:0 1, (1),  granule:1031 2, (2), 3, (3),  granule:4103 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing continuation resync in very large packets... 0, 1, 2, (2), 3, (3),  granule:4103 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('testing zero data page (1 nil packet)... 0, (0),  granule:0 1, (1),  granule:1031 2, (2),  granule:2055 ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Testing search for capture... ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('Testing recapture... ok.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # libvorbis
+    ('    vorbis_1ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('    vorbis_2ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('    vorbis_7ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    ('    vorbis_8ch_q-0.5_44100.ogg : ok',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # pth
+    ('OK - ALL TESTS SUCCESSFULLY PASSED.',
+     [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]),
+    # casync (uses meson / ninja)
+    ("ninja: Entering directory `builddir'\n"
+     '[0/1] /usr/bin/python3 -u /usr/bin/meson test --no-rebuild --print-errorlogs\n'
+     ' 1/16 test-script.sh                          OK      21.20 s \n'
+     ' 2/16 test-script-sha256.sh                   OK      23.13 s \n'
+     ' 3/16 test-script-gzip.sh                     OK      20.91 s \n'
+     ' 4/16 test-script-xz.sh                       OK      29.97 s \n'
+     ' 5/16 test-nbd.sh                             OK       0.91 s \n'
+     ' 6/16 test-fuse.sh                            OK       1.25 s \n'
+     ' 7/16 test-cachunk                            OK       0.02 s \n'
+     ' 8/16 test-cachunker                          OK       0.70 s \n'
+     ' 9/16 test-cachunker-histogram                OK       2.04 s \n'
+     '10/16 test-cadigest                           OK      10.01 s \n'
+     '11/16 test-caencoder                          OK       0.05 s \n'
+     '12/16 test-camakebst                          OK       3.21 s \n'
+     '13/16 test-caorigin                           OK       0.00 s \n'
+     '14/16 test-casync                             OK       0.74 s \n'
+     '15/16 test-cautil                             OK       0.00 s \n'
+     '16/16 test-util                               OK       0.01 s \n'
+     '\n'
+     'OK:        16\n'
+     'FAIL:       0\n'
+     'SKIP:       0\n'
+     'TIMEOUT:    0\n',
+     [16, 16, 0, 0, 0, 0, 0, 0, 0, 0]),
+    # gstreamer (uses meson / ninja)
+    ('+ meson test -C builddir\n'
+     "ninja: Entering directory `builddir'\n"
+     'ninja: no work to do.\n'
+     '1/6 gst_gst                                  OK       0.40 s\n'
+     '2/6 gst_gstabi                               FAIL     0.35 s\n'
+     '3/6 pipelines_stress                         OK       10.49 s\n'
+     '4/6 generic_sinks                            EXPECTEDFAIL 4.12 s\n'
+     '5/6 gst_gstcpp                               OK       0.37 s\n'
+     '6/6 libs_gstlibscpp                          OK       0.03 s\n'
+     'Ok:                   4\n'
+     'Expected Fail:        1\n'
+     'Fail:                 1\n'
+     'Unexpected Pass:      0\n'
+     'Skipped:              0\n'
+     'Timeout:              0\n',
+     [6, 4, 1, 1, 0, 0, 0, 0, 0, 0]),
+]
+
+backup_zero_test_data = count.zero_test_data
+
+def mock_zero_test_data():
+    pass
+
+
+class TestCount(unittest.TestCase):
+
+    def setUp(self):
+        count.zero_test_data()
+
+
+def test_generator(line, expected):
+    """
+    Generate a test for each line passed in
+    """
+    def test_parse_log(self):
+        """
+        test parse_log
+        expected = [total_tests,
+                    total_pass,
+                    total_fail,
+                    total_xfail,
+                    total_skip,
+                    counted_tests,
+                    counted_pass,
+                    counted_fail,
+                    counted_xfail,
+                    counted_skip]
+        """
+        content = '+ make check\n' + line
+        m_open = mock_open(read_data=content)
+        with patch('count.util.open_auto', m_open, create=True):
+            count.zero_test_data = mock_zero_test_data
+            count.parse_log('log')
+            count.zero_test_data = backup_zero_test_data
+
+        actual = [count.total_tests,
+                  count.total_pass,
+                  count.total_fail,
+                  count.total_xfail,
+                  count.total_skip,
+                  count.counted_tests,
+                  count.counted_pass,
+                  count.counted_fail,
+                  count.counted_xfail,
+                  count.counted_skip]
+
+        self.assertEqual(actual, expected)
+
+    return test_parse_log
+
+
+def test_setup():
+    for i, pat in enumerate(pats):
+        test_name = 'test_pat{}'.format(pat[0])
+        test = test_generator(pat[0], pat[1])
+        setattr(TestCount, test_name, test)
+
+
+# Run test_setup() to generate tests
+test_setup()
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=True)

tests/test_download.py

@@ -0,0 +1,164 @@
+from enum import Enum, auto
+import unittest
+from unittest.mock import patch, mock_open, call
+
+import pycurl
+
+import download
+
+
+class MockOpts(Enum):
+    URL = auto()
+    WRITEDATA = auto()
+    POSTFIELDS = auto()
+    FOLLOWLOCATION = auto()
+    FAILONERROR = auto()
+    CONNECTTIMEOUT = auto()
+    TIMEOUT = auto()
+    LOW_SPEED_LIMIT = auto()
+    LOW_SPEED_TIME = auto()
+
+
+def init_curl_instance(mock_curl):
+    instance = mock_curl.return_value
+    instance.URL = MockOpts.URL
+    instance.FOLLOWLOCATION = MockOpts.FOLLOWLOCATION
+    instance.FAILONERROR = MockOpts.FAILONERROR
+    instance.WRITEDATA = MockOpts.WRITEDATA
+    instance.POSTFIELDS = MockOpts.POSTFIELDS
+    return instance
+
+
+def test_opts(*opts):
+    if not opts:
+        raise Exception("no curl options specified")
+    if len(opts) != 2:
+        raise Exception("expected two args to setopt()")
+    key, val = opts
+    if key == MockOpts.WRITEDATA:
+        val.write(b'foobar')
+
+
+class TestDownload(unittest.TestCase):
+
+    @patch('download.pycurl.Curl')
+    def test_download_get_success_no_dest(self, test_curl):
+        """
+        Test successful GET request when dest is not set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo")
+        self.assertEqual(b'foobar', data.getvalue())
+
+    @patch('download.pycurl.Curl')
+    def test_download_set_basic(self, test_curl):
+        """
+        Test curl option settings set by default
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo")
+        calls = [
+            call().setopt(MockOpts.URL, 'foo'),
+            call().setopt(MockOpts.FOLLOWLOCATION, True),
+            call().setopt(MockOpts.FAILONERROR, True),
+        ]
+        test_curl.assert_has_calls(calls)
+
+    @patch('download.pycurl.Curl')
+    def test_download_set_post(self, test_curl):
+        """
+        Test setting of POSTFIELDS curl option
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo", post='postdata')
+        calls = [
+            call().setopt(MockOpts.POSTFIELDS, 'postdata'),
+        ]
+        test_curl.assert_has_calls(calls)
+
+    @patch('download.pycurl.Curl')
+    def test_download_get_failure_no_dest(self, test_curl):
+        """
+        Test failed GET request when dest is not set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        instance.perform.side_effect = pycurl.error
+        data = download.do_curl("foo")
+        self.assertIsNone(data)
+
+    @patch('download.sys.exit')
+    @patch('download.pycurl.Curl')
+    def test_download_get_failure_fatal(self, test_curl, test_exit):
+        """
+        Test failed GET request when is_fatal is set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        instance.perform.side_effect = pycurl.error
+        data = download.do_curl("foo", is_fatal=True)
+        test_exit.assert_called_once_with(1)
+
+    @patch('download.open', new_callable=mock_open)
+    @patch('download.pycurl.Curl')
+    def test_download_get_success_dest(self, test_curl, test_open):
+        """
+        Test successful GET request when dest is set.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        data = download.do_curl("foo", "testdest")
+        test_open.assert_called_once_with('testdest', 'wb')
+        test_open().write.assert_called_once_with(b'foobar')
+
+    @patch('download.os.path.exists')
+    @patch('download.open', new_callable=mock_open)
+    @patch('download.pycurl.Curl')
+    def test_download_get_write_fail_dest(self, test_curl, test_open, test_path):
+        """
+        Test failure to write to dest after successful GET request.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        test_open.side_effect = IOError
+        test_path.return_value = None
+        data = download.do_curl("foo", "testdest")
+        self.assertIsNone(data)
+
+    @patch('download.sys.exit')
+    @patch('download.os.path.exists')
+    @patch('download.open')
+    @patch('download.pycurl.Curl')
+    def test_download_write_fail_fatal(self, test_curl, test_open, test_path, test_exit):
+        """
+        Test fatal failure to write to dest after successful GET request.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        test_open.side_effect = IOError
+        test_path.return_value = None
+        data = download.do_curl("foo", "testdest", is_fatal=True)
+        test_exit.assert_called_once_with(1)
+
+    @patch('download.os.unlink')
+    @patch('download.os.path.exists')
+    @patch('download.open')
+    @patch('download.pycurl.Curl')
+    def test_download_write_fail_remove_dest(self, test_curl, test_open, test_path, test_unlink):
+        """
+        Test removal of dest following a write failure.
+        """
+        instance = init_curl_instance(test_curl)
+        instance.setopt.side_effect = test_opts
+        test_open.side_effect = IOError
+        test_path.return_value = True
+        data = download.do_curl("foo", "testdest")
+        test_path.assert_called_once_with("testdest")
+        test_unlink.assert_called_once_with("testdest")
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_files.py

@@ -0,0 +1,533 @@
+import unittest
+import config
+import files
+import tempfile
+import os
+from unittest.mock import call, MagicMock
+import build
+from files import FileManager
+
+
+def mock_return(retval):
+    """
+    Simple mock method to set return value of a function
+    """
+    def mock_fn(_):
+        return retval
+
+    return mock_fn
+
+
+class TestFiles(unittest.TestCase):
+
+    def setUp(self):
+        conf = config.Config("")
+        pkg = build.Build()
+        self.fm = FileManager(conf, pkg)
+
+    def test_banned_path(self):
+        """
+        Test banned paths are detected
+        """
+        bad_paths = ["/etc/one",
+                     "/opt/two",
+                     "/V3/opt/two",
+                     "/V4/opt/two",
+                     "/VA/opt/two",
+                     "/usr/etc/three",
+                     "/usr/local/four",
+                     "/usr/src/five",
+                     "/var/six"]
+        for path in bad_paths:
+            self.assertTrue(FileManager.banned_path(path))
+
+    def test_push_package_file(self):
+        """
+        Test push_package_file with no package name specified (package name
+        should default to 'main'
+        """
+        self.assertFalse(self.fm.newfiles_printed)
+        self.fm.push_package_file('test-fn')
+        self.assertEqual(self.fm.packages['main'], set(['test-fn']))
+        self.assertTrue(self.fm.newfiles_printed)
+
+    def test_push_package_file_banned(self):
+        """
+        Test push_package_file with banned filename'
+        """
+        self.assertFalse(self.fm.newfiles_printed)
+        self.fm.push_package_file('/etc/test-fn')
+        self.fm.push_package_file('/V3/etc/test-fn')
+        self.fm.push_package_file('/V4/etc/test-fn')
+        self.fm.push_package_file('/VA/etc/test-fn')
+        self.assertTrue(self.fm.has_banned)
+        self.assertFalse(self.fm.newfiles_printed)
+
+    def test_push_package_file_dev(self):
+        """
+        Test push_package_file with dev package specified
+        """
+        self.fm.push_package_file('test-fn', 'dev')
+        self.assertEqual(self.fm.packages['dev'], set(['test-fn']))
+        self.assertTrue(self.fm.newfiles_printed)
+
+    def test_compat_exclude_keep_file(self):
+        """
+        Test compat_exclude with a file that shouldn't be excluded.
+        """
+        self.fm.config.config_opts['compat'] = True
+        self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so.1'))
+
+    def test_compat_exclude_exclude_file(self):
+        """
+        Test compat_exclude with a file that should be excluded.
+        """
+        self.fm.config.config_opts['compat'] = True
+        self.assertTrue(self.fm.compat_exclude('/usr/lib64/libfoo.so'))
+        self.assertTrue(self.fm.compat_exclude('/V3/usr/lib64/libfoo.so'))
+        self.assertTrue(self.fm.compat_exclude('/V4/usr/lib64/libfoo.so'))
+        self.assertTrue(self.fm.compat_exclude('/VA/usr/lib64/libfoo.so'))
+
+    def test_compat_exclude_not_compat_mode(self):
+        """
+        Test compat_exclude with a file that should be excluded but isn't
+        because the package isn't being run in compat mode.
+        """
+        self.fm.config.config_opts['compat'] = False
+        self.assertFalse(self.fm.compat_exclude('/usr/lib64/libfoo.so'))
+        self.assertFalse(self.fm.compat_exclude('/V3/usr/lib64/libfoo.so'))
+        self.assertFalse(self.fm.compat_exclude('/V4/usr/lib64/libfoo.so'))
+        self.assertFalse(self.fm.compat_exclude('/VA/usr/lib64/libfoo.so'))
+
+    def test_file_pat_match(self):
+        """
+        Test file_pat_match with good match and no replacement or prefix
+        specified.
+        """
+        self.fm.push_package_file = MagicMock()
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/V3/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/V4/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_called_with('/VA/test-fn', 'main')
+
+    def test_file_pat_match_exclude(self):
+        """
+        Test file_pat_match with good match and filename in excludes list.
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.excludes.append('/test-fn')
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main'))
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'^/test-fn', 'main'))
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'^/test-fn', 'main'))
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'^/test-fn', 'main'))
+        self.fm.push_package_file.assert_not_called()
+
+    def test_file_pat_match_replacement(self):
+        """
+        Test file_pat_match with replacement provided
+        """
+        self.fm.push_package_file = MagicMock()
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/testfn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V3/testfn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V4/testfn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/VA/testfn', 'main')
+
+    def test_file_pat_match_replacement_no_glob(self):
+        """
+        Test file_pat_match with replacement provided but no glob set
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.config.config_opts['no_glob'] = True
+        self.assertTrue(self.fm.file_pat_match('/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V3/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V3/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/V4/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/V4/test-fn', 'main')
+        self.assertTrue(self.fm.file_pat_match('/VA/test-fn', r'^/test-fn', 'main', '/testfn'))
+        self.fm.push_package_file.assert_called_with('/VA/test-fn', 'main')
+
+    def test_file_windows_exe_not_allowed(self):
+        """
+        Test that Windows exe and dll files are excluded
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.config.config_opts['allow_exe'] = False
+        self.fm.push_file('/usr/bin/foo.exe', 'test')
+        self.fm.push_file('/usr/lib64/foo.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', 'test')
+        self.assertIn('/usr/bin/foo.exe', self.fm.excludes)
+        self.assertIn('/usr/lib64/foo.dll', self.fm.excludes)
+        self.assertIn('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', self.fm.excludes)
+        self.assertIn('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', self.fm.excludes)
+
+    def test_file_windows_exe_allowed(self):
+        """
+        Test that Windows exe and dll files are not excluded
+        """
+        self.fm.push_package_file = MagicMock()
+        self.fm.config.config_opts['allow_exe'] = True
+        self.fm.push_file('/usr/bin/foo.exe', 'test')
+        self.fm.push_file('/usr/lib64/foo.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', 'test')
+        self.fm.push_file('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', 'test')
+        self.assertIn('/usr/bin/foo.exe', self.fm.files)
+        self.assertIn('/usr/lib64/foo.dll', self.fm.files)
+        self.assertIn('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', self.fm.files)
+        self.assertIn('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', self.fm.files)
+        self.assertNotIn('/usr/bin/foo.exe', self.fm.excludes)
+        self.assertNotIn('/usr/lib64/foo.dll', self.fm.excludes)
+        self.assertNotIn('/usr/lib/python3.12/site-packages/nsist/msvcrt/x86/api-ms-win-core-console-l1-1-0.dll', self.fm.excludes)
+        self.assertNotIn('/usr/lib/python3.12/site-packages/installer/_scripts/t32.exe', self.fm.excludes)
+
+    def test_file_pat_match_no_match(self):
+        """
+        Test file_pat_match with no match
+        """
+        self.fm.push_package_file = MagicMock()
+        self.assertFalse(self.fm.file_pat_match('test-fn', r'testfn', 'main'))
+        self.fm.push_package_file.assert_not_called()
+
+    def test_file_is_locale(self):
+        """
+        Test file_is_locale with locale filename not present in locale list
+        """
+        self.assertEqual(self.fm.locales, [])
+        self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo'))
+        self.assertEqual(self.fm.locales, ['loc'])
+
+    def test_file_is_locale_non_locale(self):
+        """
+        Test file_is_locale with non-locale filename
+        """
+        self.assertFalse(self.fm.file_is_locale('test-fn'))
+        self.assertEqual(self.fm.locales, [])
+
+    def test_file_is_locale_present(self):
+        """
+        Test file_is_locale with locale present in locale list
+        """
+        self.fm.locales.append('loc')
+        self.assertEqual(self.fm.locales, ['loc'])
+        self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo'))
+        self.assertEqual(self.fm.locales, ['loc'])
+
+    def test_push_file_autostart(self):
+        """
+        Test push_file to autostart package, this excludes the file.
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        autostart = '/usr/lib/systemd/system/some.target.wants/some'
+        self.fm.push_file(autostart, '')
+        calls = [call(autostart, 'autostart'), call('%exclude ' + autostart, 'services')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_file_custom_extras(self):
+        """
+        Test push_file to a custom extras package
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': ["/foobar"]}}
+        self.fm.push_file('/foobar', '')
+        calls = [call('/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/foobar', '')
+        calls = [call('/V3/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/foobar', '')
+        calls = [call('/V4/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/foobar', '')
+        calls = [call('/VA/foobar', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_empty(self):
+        """
+        Test push_file with glob extras empty match
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*rightglob']]}}
+        self.fm.push_file('/leftglobrightglob', '')
+        calls = [call('/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobrightglob', '')
+        calls = [call('/V3/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobrightglob', '')
+        calls = [call('/V4/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobrightglob', '')
+        calls = [call('/VA/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_left_match(self):
+        """
+        Test push_file with glob extras left content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*']]}}
+        self.fm.push_file('/leftglobrightglob', '')
+        calls = [call('/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobrightglob', '')
+        calls = [call('/V3/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobrightglob', '')
+        calls = [call('/V4/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobrightglob', '')
+        calls = [call('/VA/leftglob*', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_right_match(self):
+        """
+        Test push_file with glob extras right content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', '*rightglob']]}}
+        self.fm.push_file('/leftglobrightglob', '')
+        calls = [call('/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobrightglob', '')
+        calls = [call('/V3/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobrightglob', '')
+        calls = [call('/V4/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobrightglob', '')
+        calls = [call('/VA/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_leftright_match(self):
+        """
+        Test push_file with glob extras leftright content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*rightglob']]}}
+        self.fm.push_file('/leftglobstuffrightglob', '')
+        calls = [call('/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobstuffrightglob', '')
+        calls = [call('/V3/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobstuffrightglob', '')
+        calls = [call('/V4/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobstuffrightglob', '')
+        calls = [call('/VA/leftglob*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_package_file_glob_multi_match(self):
+        """
+        Test push_file with multiple globs extras content matching
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.file_maps = {'foobar-extras': {'files': [['', 'leftglob*', '*rightglob']]}}
+        self.fm.push_file('/leftglobstuff/stuffrightglob', '')
+        calls = [call('/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V3/leftglobstuff/stuffrightglob', '')
+        calls = [call('/V3/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/V4/leftglobstuff/stuffrightglob', '')
+        calls = [call('/V4/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+        self.fm.push_file('/VA/leftglobstuff/stuffrightglob', '')
+        calls = [call('/VA/leftglob*/*rightglob', 'foobar-extras')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+    def test_push_file_setuid(self):
+        """
+        Test push_file with fname in setuid list
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.setuid.append('test')
+        self.fm.push_file('test', '')
+        calls = [call('%attr(4755, root, root) test', 'setuid')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+
+    def test_push_file_match(self):
+        """
+        Test push_file with match in pattern list
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.push_file('/usr/bin/test', '')
+        self.fm.push_package_file.assert_called_once_with('/usr/bin/test', 'bin')
+        self.fm.push_file('/V3/usr/bin/test', '')
+        self.fm.push_package_file.assert_has_calls([call('/V3/usr/bin/test', 'bin')])
+        self.fm.push_file('/V4/usr/bin/test', '')
+        self.fm.push_package_file.assert_has_calls([call('/V4/usr/bin/test', 'bin')])
+        self.fm.push_file('/VA/usr/bin/test', '')
+        self.fm.push_package_file.assert_has_calls([call('/VA/usr/bin/test', 'bin')])
+
+    def test_push_file_match_pkg_name_dependency(self):
+        """
+        Test push_file with match in the list on the single item that is
+        dependent on the pkg_name.
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.push_file('/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_called_once_with('/usr/share/doc/testball/*', 'doc')
+        self.fm.push_file('/V3/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_has_calls([call('/V3/usr/share/doc/testball/*', 'doc')])
+        self.fm.push_file('/V4/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_has_calls([call('/V4/usr/share/doc/testball/*', 'doc')])
+        self.fm.push_file('/VA/usr/share/doc/testball/', 'testball')
+        self.fm.push_package_file.assert_has_calls([call('/VA/usr/share/doc/testball/*', 'doc')])
+
+    def test_push_file_no_match(self):
+        """
+        Test push_file with no pattern match on the file name. Should just push
+        the unmodified filename once.
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.push_file('doesntmatcha thing', '')
+        self.fm.push_package_file.assert_called_once_with('doesntmatcha thing')
+
+    def test_remove_file(self):
+        """
+        Test remove_file with filename in files list and main package
+        """
+        self.fm.files.add('test')
+        self.fm.packages['main'] = ['test']
+        self.assertIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.files_blacklist)
+        self.assertIn('test', self.fm.packages['main'])
+        self.fm.remove_file('test')
+        self.assertNotIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.packages['main'])
+        self.assertIn('test', self.fm.files_blacklist)
+
+    def test_remove_file_not_present(self):
+        """
+        Test remove_file with filename not in files list.
+        """
+        self.assertNotIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.files_blacklist)
+        self.fm.remove_file('test')
+        self.assertNotIn('test', self.fm.files)
+        self.assertNotIn('test', self.fm.files_blacklist)
+
+    def test_clean_directories(self):
+        """
+        Test clean_directories with a directory in the list
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("/directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"], set(["/file1", "/file2"]))
+
+
+    def test_clean_directories_with_dir(self):
+        """
+        Test clean_directories with a %dir directory in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%dir /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%dir /directory", "/file1", "/file2"]))
+
+
+    def test_clean_directories_with_symlink_to_dir(self):
+        """
+        Test clean_directories with a symlink to a directory in the list. The
+        symlink should remain, but the directory should be cleaned.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            dirname = os.path.join(tmpd, "directory")
+            linkname = os.path.join(tmpd, "symlink")
+            os.mkdir(dirname)
+            os.symlink(dirname, linkname)
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("/directory")
+            self.fm.packages["main"].add("/symlink")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["/symlink"]))
+
+
+    def test_clean_directories_with_symlink_to_explicit_dir(self):
+        """
+        Test clean_directories with a symlink to a %dir directory in the list.
+        The symlink and directory should both remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            dirname = os.path.join(tmpd, "directory")
+            linkname = os.path.join(tmpd, "symlink")
+            os.mkdir(dirname)
+            os.symlink(dirname, linkname)
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%dir /directory")
+            self.fm.packages["main"].add("/symlink")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%dir /directory", "/symlink"]))
+
+
+    def test_clean_directories_with_doc(self):
+        """
+        Test clean_directories with a %doc directive in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%doc /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%doc /directory", "/file1", "/file2"]))
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_general.py

@@ -0,0 +1,24 @@
+import subprocess
+import unittest
+
+
+class TestGeneral(unittest.TestCase):
+
+    def test_ConfigParser_regressions(self):
+        """
+        Make sure ConfigParser is always called with the required
+        interpolation=None argument
+        """
+        grep_cmd = ["grep", "-re",
+                    "ConfigParser(.*(^interpolation=None).*)",
+                    "autospec"]
+        try:
+            output = subprocess.check_output(grep_cmd).decode('utf-8')
+        except subprocess.CalledProcessError as e:
+            output = e.output.decode('utf-8')
+
+        self.assertEqual(output.strip(), "")
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=True)

tests/test_license.py

@@ -0,0 +1,264 @@
+from contextlib import redirect_stdout
+from io import BytesIO, StringIO
+import os
+import tempfile
+import unittest
+from unittest.mock import patch, mock_open, MagicMock
+
+import pycurl
+
+import config
+import download
+import license
+import util
+
+
+class TestLicense(unittest.TestCase):
+
+    def setUp(self):
+        license.licenses = []
+
+    def test_add_license(self):
+        """
+        Test add_license from valid string, Apache-2 should be translated to
+        Apache-2.0
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        self.assertTrue(license.add_license('Apache-2', conf.license_translations, conf.license_blacklist))
+        self.assertIn('Apache-2.0', license.licenses)
+
+    def test_add_license_present(self):
+        """
+        Test add_license from valid string, but license is already present in
+        the licenses list. Should return True and should not modify the
+        licenses list. GPL-3 translates to GPL-3.0.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        license.licenses.append('GPL-3.0')
+        self.assertTrue(license.add_license('GPL-3', conf.license_translations, conf.license_blacklist))
+        self.assertEqual(['GPL-3.0'], license.licenses)
+
+    def test_add_license_blacklisted(self):
+        """
+        Test add_license from string in license_blacklist. Should return False
+        and should not modify the licenses list.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        # sanity check to make sure the licenses list is empty before the later
+        # assertIn() call
+        self.assertEqual(license.licenses, [])
+
+        self.assertFalse(license.add_license('License', conf.license_translations, conf.license_blacklist))
+        self.assertNotIn('License', license.licenses)
+
+    def test_license_from_copying_hash(self):
+        """
+        Test license_from_copying_hash with valid license file
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+        self.assertIn('GPL-3.0', license.licenses)
+
+    def test_license_from_copying_hash_no_license_show(self):
+        """
+        Test license_from_copying_hash with invalid hash and no license_show
+        set
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        # remove the hash from license_hashes
+        del(conf.license_hashes[license.get_sha1sum('tests/COPYING_TEST')])
+        conf.license_show = "license.show.url"
+        license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+
+        self.assertEqual(license.licenses, [])
+
+    def test_license_from_copying_hash_bad_license(self):
+        """
+        Test license_from_copying_hash with invalid license file
+        """
+        conf = config.Config("")
+        content = util.get_contents("tests/COPYING_TEST").replace(b"GNU", b"SNU")
+        m_open = MagicMock()
+        m_open.__str__.return_value = content
+
+        with patch('license.get_contents', m_open, create=True):
+            license.license_from_copying_hash('copying.txt', '', conf, '')
+
+        self.assertEqual(license.licenses, [])
+
+    def test_license_from_copying_hash_license_server_excep(self):
+        """
+        Test license_from_copying_hash with license server when pycurl raises
+        an exception.
+        """
+        class MockCurl():
+            URL = None
+            WRITEDATA = None
+            POSTFIELDS = None
+            FOLLOWLOCATION = 0
+            FAILONERROR = False
+            CONNECTTIMEOUT = 0
+            TIMEOUT = 0
+            LOW_SPEED_LIMIT = 0
+            LOW_SPEED_TIME = 0
+            def setopt(_, __, ___):
+                pass
+
+            def perform(_):
+                raise pycurl.error('Test Exception')
+
+            def close(_):
+                pass
+
+        # set the mock curl
+        download.pycurl.Curl = MockCurl
+
+        conf = config.Config("")
+        conf.license_fetch = 'license.server.url'
+
+        # let's check that the proper thing is being printed as well
+        out = StringIO()
+        with redirect_stdout(out):
+            with self.assertRaises(SystemExit):
+                license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+
+        self.assertIn('Unable to fetch license.server.url: Test Exception', out.getvalue())
+
+        # unset the manual mock
+        download.pycurl.Curl = pycurl.Curl
+
+    def test_license_from_copying_hash_license_server(self):
+        """
+        Test license_from_copying_hash with license server. This is heavily
+        mocked.
+        """
+        class MockBytesIO(BytesIO):
+            """
+            Mock class for BytesIO to set returnvalue of BytesIO.getvalue()
+            """
+            def getvalue(_):
+                return 'GPL-3.0'.encode('utf-8')
+
+        # set the mocks
+        download.BytesIO = MockBytesIO
+
+        class MockCurl():
+            URL = None
+            WRITEDATA = None
+            POSTFIELDS = None
+            FOLLOWLOCATION = 0
+            FAILONERROR = False
+            CONNECTTIMEOUT = 0
+            TIMEOUT = 0
+            LOW_SPEED_LIMIT = 0
+            LOW_SPEED_TIME = 0
+            def setopt(_, __, ___):
+                pass
+
+            def perform(_):
+                pass
+
+            def close(_):
+                pass
+
+            def getinfo(_, __):
+                return 200
+
+        # set the mock curl
+        download.pycurl.Curl = MockCurl
+
+        conf = config.Config("")
+        conf.license_fetch = 'license.server.url'
+
+        # let's check that the proper thing is being printed as well
+        out = StringIO()
+        with redirect_stdout(out):
+            license.license_from_copying_hash('tests/COPYING_TEST', '', conf, '')
+
+        self.assertIn('GPL-3.0', license.licenses)
+        self.assertIn('License     :  GPL-3.0  (server)', out.getvalue())
+
+        # unset the manual mock
+        download.BytesIO = BytesIO
+
+        # unset the manual mock
+        download.pycurl.Curl = pycurl.Curl
+
+    def test_scan_for_licenses(self):
+        """
+        Test scan_for_licenses in temporary directory with valid license file
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        with open('tests/COPYING_TEST', 'rb') as copyingf:
+            content = copyingf.read()
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            # create the copying file
+            with open(os.path.join(tmpd, 'COPYING'), 'w') as newcopyingf:
+                newcopyingf.write(content.decode('utf-8'))
+            # create some cruft for testing
+            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
+                with open(os.path.join(tmpd, testf), 'w') as newtestf:
+                    newtestf.write('test content')
+            license.scan_for_licenses(tmpd, conf, '')
+
+        self.assertIn('GPL-3.0', license.licenses)
+
+    def test_scan_for_licenses_none(self):
+        """
+        Test scan_for_licenses in temporary directory with no matching files.
+        Should not add any licenses, should print a fatal message, should exit
+        with a status code of 1.
+        """
+        conf = config.Config("")
+        conf.setup_patterns()
+        with tempfile.TemporaryDirectory() as tmpd:
+            # create some cruft for testing
+            for testf in ['testlib.c', 'testmain.c', 'testheader.h']:
+                with open(os.path.join(tmpd, testf), 'w') as newtestf:
+                    newtestf.write('test content')
+            # let's check that the proper thing is being printed as well
+            out = StringIO()
+            with redirect_stdout(out):
+                with self.assertRaises(SystemExit) as thread:
+                    license.scan_for_licenses(tmpd, conf, '')
+
+        self.assertEqual(thread.exception.code, 1)
+        self.assertIn("Cannot find any license", out.getvalue())
+        self.assertEqual(license.licenses, [])
+
+    def test_load_specfile(self):
+        """
+        Test load_specfile with populated license list. This method is not
+        normally tested but there is some logic here.
+        """
+        class MockSpecfile(object):
+            licenses = []
+
+        license.licenses = ['GPL-3.0', 'MIT']
+        specfile = MockSpecfile()
+        license.load_specfile(specfile)
+        self.assertEqual(specfile.licenses, license.licenses)
+
+    def test_load_specfile_none(self):
+        """
+        Test load_specfile with unpopulated license list. This method is not
+        normally tested but there is some logic here.
+        """
+        class MockSpecfile(object):
+            licenses = []
+
+        license.licenses = []
+        specfile = MockSpecfile()
+        license.load_specfile(specfile)
+        self.assertEqual(specfile.licenses, [license.default_license])
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_pkg_integrity.py

@@ -0,0 +1,450 @@
+from io import BytesIO
+import json
+import os
+import shutil
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import config
+import download
+import pkg_integrity
+
+
+TESTDIR = os.path.join(os.getcwd(), "tests/testfiles/pkg_integrity")
+TESTKEYDIR = os.path.join(TESTDIR, "testkeys")
+
+PACKAGE_URL = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz"
+NO_SIGN_PKT_URL = "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz"
+NOSIGN_PKT_URL_BAD = "http://gnu.mirrors.pair.com/savannah/savannah/quagga/bad_quagga-1.1.0.tar.gz"
+NOSIGN_PKT_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz"
+NOSIGN_SIGN_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz.asc"
+GNOME_SHA256_PKG = "https://download.gnome.org/sources/pygobject/3.24/pygobject-3.24.0.tar.xz"
+QT_SHA256_PKG = "https://download.qt.io/official_releases/qt/5.12/5.12.4/submodules/qtspeech-everywhere-src-5.12.4.tar.xz"
+KEYID = "EC2392F2EDE74488680DA3CF5F2B4756ED873D23"
+
+
+def mock_download_do_curl(url, dst=None):
+    bad_sigs = ["http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.asc",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.sign"]
+    if not dst:
+        return BytesIO(b'foobar')
+    src = os.path.join(TESTDIR, os.path.basename(url))
+    if dst and os.path.isfile(src):
+        shutil.copyfile(src, dst)
+        return dst
+    else:
+        return None
+    if url in bad_sigs:
+        return None
+
+
+@patch('download.do_curl', mock_download_do_curl)
+class TestCheckFn(unittest.TestCase):
+
+    def test_check_matching_sign_url(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            pkey = "023A4420C7EC6914.pkey"
+            shutil.copy(os.path.join(TESTKEYDIR, pkey), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd)
+            result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertTrue(result)
+
+    def test_check_with_existing_sign(self):
+        """ Download signature for local verification """
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd)
+            result = pkg_integrity.check(NOSIGN_PKT_URL, conf)
+            self.assertTrue(result)
+
+
+@patch('download.do_curl', mock_download_do_curl)
+class TestDomainBasedVerifiers(unittest.TestCase):
+
+    def run_test_for_domain(self, Verifier, url):
+        with tempfile.TemporaryDirectory() as tmpd:
+            filen = os.path.basename(url)
+            shutil.copy(os.path.join(TESTDIR, filen), tmpd)
+            package_path = os.path.join(tmpd, filen)
+            verifier = Verifier(**{'package_path': package_path,
+                                   'url': url})
+            return verifier.verify()
+        return None
+
+    def _mock_fetch_shasum(url):
+        return (
+                "100395496483fcea7ba03fc1655c7a770f7f2e12e93be8bda2e31fec42debde0 pygobject-3.24.0.news\n"
+                "ae417db3be2a197b403bba6472cfb35a6e642cd802660832acb9c96123f79463  pygobject-3.24.0.changes\n"
+                "4e228b1c0f36e810acd971fad1c7030014900d8427c308d63a560f3f1037fa3c pygobject-3.24.0.tar.xz"
+                )
+
+    @patch('pkg_integrity.GnomeOrgVerifier.fetch_shasum', _mock_fetch_shasum)
+    def test_gnome_org(self):
+        result = self.run_test_for_domain(pkg_integrity.GnomeOrgVerifier, GNOME_SHA256_PKG)
+        self.assertTrue(result)
+
+    @patch('pkg_integrity.QtIoVerifier.fetch_shasum')
+    def test_qt_io(self, test_fetch):
+        test_fetch.return_value = "2ff9660fb3f5663c9161f491d1a304db62691720136ae22c145ef6a1c94b90ec  qtspeech-everywhere-src-5.12.4.tar.xz\n"
+        result = self.run_test_for_domain(pkg_integrity.QtIoVerifier, QT_SHA256_PKG)
+        self.assertTrue(result)
+
+
+@patch('download.do_curl', mock_download_do_curl)
+class TestGPGVerifier(unittest.TestCase):
+
+    def test_from_url(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertTrue(result)
+
+    def test_invalid_key(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd)
+            with open(os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD) + ".asc"), 'w') as ofile:
+                ofile.write("Invalid signature")
+            result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, conf)
+            self.assertIsNone(result)
+
+    def test_key_not_found(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "6FE57CA8C1A4AEA6.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL_BAD)), tmpd)
+            result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, conf)
+            self.assertIsNone(result)
+
+    def test_from_disk(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
+            out_key = out_file + ".asc"
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)) + ".asc", tmpd)
+            result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key, conf)
+            self.assertTrue(result)
+
+    def test_non_matchingsig(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
+            f = open(out_file, 'wb')
+            f.write(b'made up date that will fail check')
+            f.close()
+            with self.assertRaises(SystemExit) as a:
+                pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertEqual(a.exception.code, 1)
+
+    def test_result_on_non_existent_pkg_path(self):
+        conf = config.Config('')
+        conf.rewrite_config_opts = unittest.mock.Mock()
+        conf.config_opts['verify_required'] = False
+        result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz',
+                                         'NonExistentPKG.tar.gz',
+                                         'NonExistentKey.asc',
+                                         conf)
+        self.assertIsNone(result)
+
+    def test_result_on_nosign_package(self):
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(NOSIGN_PKT_URL)), tmpd)
+            result = pkg_integrity.check(NO_SIGN_PKT_URL, conf)
+            self.assertIsNone(result)
+
+    @patch.object(pkg_integrity.GPGCli, 'exec_cmd')
+    @patch('pkg_integrity.parse_gpg_packets')
+    def test_result_multiple_sig(self, mock_parse, mock_exec):
+        """Test verification of first signature of tarball with multiple signatures."""
+        def packets_separator(filename, **kwargs):
+            if filename.endswith('.pkey'):
+                packets = [
+                    {
+                        'offset': 528,
+                        'length': 37,
+                        'type': 'user ID',
+                        'email': 'user1@example.com',
+                    },
+                ]
+            elif filename.endswith('.asc'):
+                packets = [
+                    {
+                        'offset': 0,
+                        'length': 543,
+                        'type': 'signature',
+                        'keyid': '023A4420C7EC6914',
+                    },
+                    {
+                        'offset': 543,
+                        'length': 543,
+                        'type': 'signature',
+                        'keyid': '12345678DEADCAFE',
+                    },
+                ]
+            return packets
+        mock_parse.side_effect = packets_separator
+
+        mock_exec.return_value = (b'', b'', 0)
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertTrue(result)
+            self.assertEqual(mock_parse.call_count, 4)
+            self.assertEqual(mock_exec.call_count, 3)
+            self.assertEqual(pkg_integrity.EMAIL, "user1@example.com")
+            self.assertEqual(pkg_integrity.KEYID, "023A4420C7EC6914")
+
+    @patch.object(pkg_integrity.GPGCli, 'exec_cmd')
+    @patch('pkg_integrity.parse_gpg_packets')
+    def test_result_multiple_sig_no_separators(self, mock_parse, mock_exec):
+        """Test skipping of sig verification in the multiple sig case when packet separators are absent."""
+        def packets_no_separator(filename, **kwargs):
+            if filename.endswith('.pkey'):
+                packets = [
+                    {
+                        'type': 'user ID',
+                        'email': 'user2@example.com',
+                    },
+                ]
+            elif filename.endswith('.asc'):
+                packets = [
+                    {
+                        'type': 'signature',
+                        'keyid': '023A4420C7EC6914',
+                    },
+                    {
+                        'type': 'signature',
+                        'keyid': 'DEADCAFEC7EC6914',
+                    },
+                ]
+            return packets
+        mock_parse.side_effect = packets_no_separator
+
+        mock_exec.return_value = (b'', b'', 0)
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            conf = config.Config(tmpd)
+            conf.rewrite_config_opts = unittest.mock.Mock()
+            conf.config_opts['verify_required'] = False
+            shutil.copy(os.path.join(TESTKEYDIR, "023A4420C7EC6914.pkey"), tmpd)
+            shutil.copy(os.path.join(TESTDIR, os.path.basename(PACKAGE_URL)), tmpd)
+            with self.assertRaises(SystemExit) as msg:
+                result = pkg_integrity.check(PACKAGE_URL, conf)
+            self.assertEqual(msg.exception.code, 1)
+            self.assertEqual(mock_parse.call_count, 4)
+            self.assertEqual(mock_exec.call_count, 2)
+            self.assertEqual(pkg_integrity.EMAIL, "user2@example.com")
+            self.assertEqual(pkg_integrity.KEYID, "023A4420C7EC6914")
+
+
+class TestInputGetter(unittest.TestCase):
+
+    def test_timput(self):
+        ig = pkg_integrity.InputGetter(default='N', timeout=2)
+        answer = ig.get_answer()
+        self.assertIsNone(answer)
+        ig = pkg_integrity.InputGetter(default='Y', timeout=2)
+        answer = ig.get_answer()
+        self.assertIsNone(answer)
+
+
+class TestUtils(unittest.TestCase):
+
+    def test_get_verifier(self):
+        x = pkg_integrity.get_verifier('file.abcd')
+        self.assertEqual(x, None)
+
+        y = pkg_integrity.get_verifier('xorriso-1.4.6.tar.gz')(
+                package_path='',
+                url='http://ftp.gnu.org/gnu/xorriso/xorriso-1.4.6.tar.gz',
+                package_check='http://ftp.gnu.org/gnu/xorriso/xorriso-1.4.6.tar.gz.asc'
+                )
+        self.assertTrue(isinstance(y, pkg_integrity.GPGVerifier))
+
+    def test_parse_gpg_packets_for_keyid(self):
+        """Test parse_gpg_packets() to retrieve keyid info."""
+        def check_packets(algo, key_id, packet_count, packet_with_val):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                packets = pkg_integrity.parse_gpg_packets(tmpf.name)
+                self.assertIsNotNone(packets)
+                self.assertEqual(len(packets), packet_count)
+                self.assertEqual(packets[packet_with_val]["keyid"], key_id)
+                tmpf.close()
+
+        check_packets(KEY_ALGO17, '8AFAFCD242818A52', 6, 1)
+        check_packets(KEY_ALGO1, '330239C1C4DAFEE1', 1, 0)
+
+    def test_get_keyid(self):
+        """Test get_keyid() to retrieve key ID from GPG key or signature."""
+        def check_get_keyid(algo, key_id):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                result = pkg_integrity.get_keyid(tmpf.name)
+                self.assertEqual(result, key_id)
+                tmpf.close()
+
+        check_get_keyid(KEY_ALGO17, '8AFAFCD242818A52')
+        check_get_keyid(KEY_ALGO1, '330239C1C4DAFEE1')
+
+    def test_get_keyid_none(self):
+        """Test get_keyid() when the key name is invalid."""
+        false_name = '/false/name'
+        self.assertTrue(pkg_integrity.get_keyid(false_name) is None)
+
+    def _mock_download_file(url, dst=None):
+        # make return codes match by url to ensure we are using the expected signature type
+        if url in ("http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz.sig",
+                   "http://download.savannah.gnu.org/releases/quilt/quilt-0.65.tar.gz.asc",
+                   "http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2.sign"):
+            return os.path.join(dst, os.path.basename(url))
+        return None
+
+    @patch('download.do_curl', _mock_download_file)
+    def test_get_signature_url(self):
+        url_from_gnu = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz"
+        url_from_gnu1 = "http://download.savannah.gnu.org/releases/quilt/quilt-0.65.tar.gz"
+        url_from_gnu2 = "http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2"
+
+        self.assertEqual(pkg_integrity.get_signature_file(url_from_gnu, '.')[-4:], '.sig')
+        self.assertEqual(pkg_integrity.get_signature_file(url_from_gnu1, '.')[-4:], '.asc')
+        self.assertEqual(pkg_integrity.get_signature_file(url_from_gnu2, '.')[-5:], '.sign')
+
+    def test_parse_gpg_packets_for_email(self):
+        """Test parse_gpg_packets() to retrieve email info."""
+        def check_packets(algo, email, packet_count, packet_with_val):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                packets = pkg_integrity.parse_gpg_packets(tmpf.name)
+                self.assertIsNotNone(packets)
+                self.assertEqual(len(packets), packet_count)
+                if packet_with_val:
+                    self.assertEqual(packets[packet_with_val]["email"], email)
+                tmpf.close()
+
+        check_packets(KEY_ALGO17, 'kislyuk@gmail.com', 6, 0)
+        check_packets(KEY_ALGO1, None, 1, None)
+
+    def test_get_email(self):
+        """Test get_email() to retrieve email info from GPG key."""
+        def check_get_email(algo, email):
+            with tempfile.NamedTemporaryFile(delete=True) as tmpf:
+                tmpf.write(algo)
+                tmpf.flush()
+                result = pkg_integrity.get_email(tmpf.name)
+                self.assertEqual(result, email)
+                tmpf.close()
+
+        check_get_email(KEY_ALGO17, 'kislyuk@gmail.com')
+        check_get_email(KEY_ALGO1, None)
+
+KEY_ALGO1 = b"""\
+-----BEGIN PGP SIGNATURE-----
+
+iQEcBAABAgAGBQJX1yrCAAoJEDMCOcHE2v7hwhcH/AqhJ/vqkCOo09Yh89bYRFyb
+O5yEpaRV7vo4qoqXht3fQiR3KC3lSKybtScihbQ0xcTeBzSwGPMKPRpOqXEKRnwY
+9Zq8ev72Ixi5yVsKdSKjoeM4smXJdQolKnrKy0chsOMzu7cxk7hwejplIMjycKza
+g3HM6jtw6v10JDj6a5SJPkrufj2eIHq1enn2di9q9+yyAJGiuWi3ABfINmL3Y9sa
+zODoMBD/B7LCZ1Zv6dG7kaN1XS82crOmXrtgcWslKxfGPkdW9SGQdhCtm6f/Z5/w
+29adzXFicobZBFBHVnbQ0iRf39omkNxPOhMYwLVQFprzjOPm/DDHSztZj87jbOM=
+=Qe1y
+-----END PGP SIGNATURE-----
+"""
+
+KEY_ALGO17 = b"""\
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQGiBEYdIyURBACrG5G9L7R2uLGnCer+liVhsn5vwJIwIpLtVTb7Z3AcOoumza66
+5JBZtY9RSsNRbMcuqxPWeXnl8jhs513O0dZihTL+/cLMD1RJAXRlf0eDxAYl2TiD
+7AdH+y6HGHljn2IkH5jIzdTgNXdJZ8BArkFZP7an+ZLfE5RgSQ4QqZpuxwCgiq9+
+fjVoHSvGJHlcN4SiBBn8cCkEAKsYT7rp7N2GaA6b4XjLr2bVnv3MV431EhxTHynX
+mcnpT91ItMAkjPWcMSWWBhYJ/rgi2KYh5oAMhcLt2gCU58TxMsxA2rPEocHFDzKY
+Yj9HOXCa4XasL4R+Jyu4/Flh7Fqvrts5iZViDx/hPo1O5HhM7VuVnDxOc6yVPDB1
+OGj5A/9lCCn2awr4XxEWsYnBDtoqzfb8jwIDfS4xtqCAaegiXr4P1GGgkDtlW0XC
+5k2giFiqD0l928y7YV7Mumw2hjqECyPwDDbsEMfJasWEnAG7hBmAfHtIwoC8563d
+3QM16Wu33xXB+rbss/mH95oUOEtkR1m9HCAOZ+R/T9WVdwTKN7QiQW5kcmV5IEtp
+c2x5dWsgPGtpc2x5dWtAZ21haWwuY29tPohoBBMRAgAoBQJXdAKuAhsDBQkSzAMA
+BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCK+vzSQoGKUgbWAJsFQ1PeMKGA
+JC3GyQqoKIN65KibwQCfVWLmpSuP9Zmb9prjPDQyt8B0NXG0MkFuZHJleSBLaXNs
+eXVrICh3ZWF2ZXIpIDxraXNseXVrQG9jZi5iZXJrZWxleS5lZHU+iGYEExECACYF
+AkYdIyUCGwMFCRLMAwAGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCK+vzSQoGK
+UiE9AKCADVzPhdseTQbrsVfZtqgIDqC/ZgCeL3UK2vHkLHBWiYzf1mCXzfgRpiG5
+Ag0ERh0jKxAIAImRscnCUdzp52n8KLPsyD/U+hn5YAUfGef0n6+eUuEh1ygH0J+H
+8W72K16dReY9X05Htod699TpHCWhmAL9DhqbAbjRtJ5B7hoTq6IS/FWbKRnsW/x7
+il/pY6s7P74J1y1NYTbDC+5spw3QCACieU8QdK54sOhUVgTWYC55mu1ycVLSYFjp
+me7Q6nKpcEq67zHx0QLmjFEryarlL7wEBvB33KzeAQ4thif8SHn3UM5TPkGlMs0h
+T/BFNYH+dMd703HS+rwv1/vs87PxhIb+QqoK9BwVbZdLEd12D1Amk/oX/hZgELUO
+tBgMj5Zbk+Jl6AWp8RgRWnzV1JSu2Zj/1wsAAwYH/1DwGUhpu9Sg3DgeEqp4tyyB
+qVUtTXNc6F3XVzeD3i7jahjdJAlMhi+rQrQNZXNVTMdHn7mec5Oi6urGR79eabFv
+cENWDHKj6QkMo4PCIi+y7TSoLyywQLxxxsDz5FGC5eTwSzo9Cdm6nwWhmCwNR2XG
+bb68/TVinfYCXyo+q6EfmOKDeBQxCTdiUkWnkMPPfw4VV2/7KPxuJWcJ+eENFlY1
+mHONQdJneIGpnXnvIT4peBPIVH4Q8FmZOzxEBvZQPKXvE3rxZYpFQBUmTb5kCeBb
+VDRHm4iUY2IR1VFk2ExCCnYS6ds4Alt5sy4jQGY9ttOAXDX9hUBUpomG6AAvafqI
+TwQYEQIADwUCRh0jKwIbDAUJEswDAAAKCRCK+vzSQoGKUmKLAJ90lCzOl94ztnyT
+dImfPudUs7wYFQCeOS0T+YvYajEA99RWUnx2IH1rFKK5Ay4EV3QCMxEIAME79anw
++LZGB/d240txYXggyLefLskCJaIZrRoqxflTJCu6Gs6tTR8J29HO+GCuGKMMpLeF
+x9uSMU2VeO+3HlpW9MD4EPGPL1ZZGecftXwFTMLCcoiPkLGqMzidENgQpNTyKJpE
+vX1F3gIU5O53F35xeCWYSwNNAY9V+knNXG9syN59hrwa3Avr0Pu39lAABPKqKpqJ
+LLMtxPSDfJljaEpbeTr0SS18cGO5ytK+sH7jBhp5oBi79tcqV3zpDCh1crSfZewb
+Ugldem19fx3bRP7okYxSSJrDGjgte9jQWejeHRrUlSI9W8DEmVMo43CiEN6IG4aF
+fYLs1MQK25nzc58BAKK5PWll/moC9Ne3sF2Deuw91gLNLEbwlQto4M49IMCRB/9h
+9Ylaq+C4M3db46v/imwlSJmL7zVrbdENAY4aJEgpnSFNsgcFgnU0od+XTKiIVum+
+Fxr6PbB2DcAoMqeGsu6R+uAWFJiAMyqLrAkcMeYg3p6DxY17oCLcIiEm9hnlA0K8
+mYOw7Ykkb+oJqjhBUm5PFQH5KmXp/fbBuRiYrDfI/hBMw7J6XouH3vvrnIR5gBJM
+UvFkFt9RDMFQ2g+PjHbvZx1RMNMRJdw2uqAl+HHkF2a/Piyf+U6lBnKnV161FPbS
+mGKgbguN6y5SDfuDYHBH+GOmuYYDiU+BQnPhp0+fmEb2xZS4EzM7vcoCUzOAIbdl
+EV4v/tduwyLcWX+J3sIfCACxxmKDR1aKhdG6BHY2UmYIsp54xo/MOJmco2DlQUpj
+c45J8/4XYspXt5rI8fby81vHD2qMiDJ4aEYjDkmW+6XhuYjyr4xTqkvS9gMDggqH
+EccGqz7IoWrQPWewDGxVINq9rLU0l6puKR6aXwx8J9RwRYy0oECjSdETuP+NdxXd
+OVvnvBYMKSWWbIam1sjIiduczyVVRv8t1M5EKMG+KR/VS+hyjNZYduSp7OmettWT
+wyIG3LXiBHffFHPZ0kn3P/sZ8inBEzsjfiVm9mLL3yIL2C1YSY2uaiigPBPeNJM3
+QSnY6rSpfyGVKTC4/O9jX/+G0DJ7k0fdb7JwIS6Kd0huiK8EGBECAA8FAld0AjMC
+GwIFCRLMAwAAagkQivr80kKBilJfIAQZEQgABgUCV3QCMwAKCRD51uveZ3aeBPHd
+AP0WPpdXXqw2/kjuxydswKsI7I07uZlWeyou9zpath+0YgD8CW3o79TDafTHnpbo
+fX3HT+yO2BftAx+rzCymzllQLe+kcwCfeiLfkBj+ZyX9fhNT5YyiagmY5LoAn1TH
+qA34waak3JSxv16lZC0TqIuD
+=cHO/
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_specdescription.py

@@ -0,0 +1,252 @@
+import unittest
+from unittest.mock import mock_open, patch
+import config
+import specdescription
+
+
+class TestSpecdescription(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        self.open_name = 'specdescription.util.open_auto'
+
+    def setUp(self):
+        specdescription.default_description = "No detailed description available"
+        specdescription.default_description_score = 0
+        specdescription.default_summary = "No detailed summary available"
+        specdescription.default_summary_score = 0
+        specdescription.license.licenses = []
+
+    def test_clean_license_string(self):
+        """
+        Test clean_license_string with a list of license strings
+        """
+        lics = [("ZPL-2.1", "ZPL-2.1"),
+                ("Apache-2.0", "Apache-2.0"),
+                ("GPL-2.0+", "GPL-2.0+"),
+                ("GPL v2", "GPL-2"),
+                ("GPL (>= 2)", "GPL-2.0+"),
+                ("BSD 2-Clause Simplified", "BSD-2-Clause ")]
+
+        for lic, exp in lics:
+            self.assertEqual(specdescription.clean_license_string(lic), exp)
+
+    def test_description_from_spec(self):
+        """
+        Test description_from_spec with summary and description present in
+        spec file
+        """
+        content = "# this is a specfile\n"          \
+                "License: GPL v2\n"                 \
+                "Summary: This is a test package\n" \
+                "\n"                                \
+                "%description\n"                    \
+                "Here is the description section\n" \
+                "Look it continues until the\n"     \
+                "line that begins with '%'\n"       \
+                "\n"                                \
+                "%donewithdesc\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_spec("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "Here is the description section\n"
+                         "Look it continues until the\n"
+                         "line that begins with '%'\n\n")
+        self.assertEqual(specdescription.default_summary,
+                         "This is a test package\n")
+        self.assertEqual(specdescription.default_description_score, 4)
+        self.assertEqual(specdescription.default_summary_score, 4)
+        self.assertEqual(specdescription.license.licenses, ["GPL-2"])
+
+    def test_description_from_spec_no_info(self):
+        """
+        Test description_from_spec with no license, summary or description in
+        spec file
+        """
+        content = "# this is a specfile without much in it\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_spec("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "No detailed description available")
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_description_score, 0)
+        self.assertEqual(specdescription.default_summary_score, 0)
+        self.assertEqual(specdescription.license.licenses, [])
+
+    def test_description_from_pkginfo(self):
+        """
+        Test description_from_pkginfo with information in pkg-info file
+        """
+        content = "# this is a package info file\n"  \
+                "License: GPL v2\n"                  \
+                "abstract: This is a test package\n" \
+                "\n"                                 \
+                "Description:\n"                     \
+                "Here is the description section\n"  \
+                "Look it continues until the\n"      \
+                "line that contains a colon.\n"      \
+                "\n"                                 \
+                "donewithdesc:\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_pkginfo("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "Here is the description section\n"
+                         "Look it continues until the\n"
+                         "line that contains a colon.\n\n")
+        self.assertEqual(specdescription.default_summary,
+                         "This is a test package")
+        self.assertEqual(specdescription.default_description_score, 4)
+        self.assertEqual(specdescription.default_summary_score, 4)
+        self.assertEqual(specdescription.license.licenses, ["GPL-2"])
+
+    def test_description_from_pkginfo_no_info(self):
+        """
+        Test description_from_pkginfo with no information in the pkginfo file
+        """
+        content = "# this is a pkginfo file without much in it\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_pkginfo("filename", {}, [])
+
+        self.assertEqual(specdescription.default_description,
+                         "No detailed description available")
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_description_score, 0)
+        self.assertEqual(specdescription.default_summary_score, 0)
+        self.assertEqual(specdescription.license.licenses, [])
+
+    def test_summary_from_pkgconfig(self):
+        """
+        Test summary_from_pkgconfig with a summary present in the file. The
+        file is named <package>.pc, increasing the summary score
+        """
+        content = "# this is a pkgconfig file\n" \
+                  "Description: this is a test package\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("testpkg.pc", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "this is a test package\n")
+        self.assertEqual(specdescription.default_summary_score, 3)
+
+    def test_summary_from_pkgconfig_misnamed_file(self):
+        """
+        Test summary_from_pkgconfig with a summary present in the file. The
+        file is named pkg.pc, which does not increase the summary score to 3
+        """
+        content = "# this is a pkgconfig file\n" \
+                  "Description: this is a test package\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("pkg.pc", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "this is a test package\n")
+        self.assertEqual(specdescription.default_summary_score, 2)
+
+    def test_summary_from_pkgconfig_no_info(self):
+        """
+        Test summary_from_pkgconfig with no summary in the file.
+        """
+        content = "# this is a pkgconfig file without a description\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("pkg.pc", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_summary_score, 0)
+
+    def test_summary_from_R(self):
+        """
+        Test summary_from_R with DESCRIPTION file
+        """
+        content = "# this is a pkgconfig file\n" \
+                  "Title: this is a test package\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_R("DESCRIPTION")
+
+        self.assertEqual(specdescription.default_summary,
+                         "this is a test package\n")
+        self.assertEqual(specdescription.default_summary_score, 3)
+
+    def test_summary_from_R_no_info(self):
+        """
+        Test summary_from_R with no summary in the file
+        """
+        content = "# this is a DESCRIPTION file without a description\n"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.summary_from_pkgconfig("DESCRIPTION", "testpkg")
+
+        self.assertEqual(specdescription.default_summary,
+                         "No detailed summary available")
+        self.assertEqual(specdescription.default_summary_score, 0)
+
+    def test_skipline(self):
+        """
+        Test skipline for a list of lines
+        """
+        skips = ["Copyright",
+                 "Free Software Foundation, Inc.",
+                 "Copying and distribution of",
+                 "are permitted in any",
+                 "notice and this notice",
+                 "README",
+                 "-*-",
+                 "blabla introduction"]
+
+        for skip in skips:
+            self.assertTrue(specdescription.skipline(skip))
+
+        self.assertFalse(specdescription.skipline('nothing to skip here'))
+
+    def test_description_from_readme(self):
+        """
+        Test description_from_readme with greater than 80 characters and a file
+        name README, increasing the score
+        """
+        content = "This is a readme blah blah blah blah test test test\n" \
+                  "this is the package but it is just a test package\n"   \
+                  "ok?\n\n"                                               \
+                  "This is a new paragraph, shouldn't be included"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_readme("README")
+
+        self.assertEqual(specdescription.default_description,
+                         "This is a readme blah blah blah blah test test test\n"
+                         "this is the package but it is just a test package\n"
+                         "ok?\n")
+        self.assertEqual(specdescription.default_description_score, 1.5)
+
+    def test_description_from_readme_with_skiplines(self):
+        """
+        Test description_from_readme with a skipline.
+        """
+        content = "This is a readme blah blah blah blah test test test\n"  \
+                  "this is the package Copying and distribution of this\n" \
+                  "ok? More characters needed to get over 80\n\n"          \
+                  "This is a new paragraph, shouldn't be included"
+        m_open = mock_open(read_data=content)
+        with patch(self.open_name, m_open, create=True):
+            specdescription.description_from_readme("Readme.md")
+
+        self.assertEqual(specdescription.default_description,
+                         "This is a readme blah blah blah blah test test test\n"
+                         "ok? More characters needed to get over 80\n")
+        self.assertEqual(specdescription.default_description_score, 1)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_specfile.py

@@ -0,0 +1,470 @@
+import unittest
+import unittest.mock
+import buildreq
+import config
+import specfiles
+import tarball
+
+
+class TestSpecfileWrite(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        self.specfile = None
+        self.WRITES = []
+
+    def setUp(self):
+        conf = config.Config('/download/path')
+        conf.config_opts['dev_requires_extras'] = False
+        url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz"
+        content = tarball.Content(url, 'pkg', '1.0', [], conf, '/tmp')
+        conf.content = content
+        reqs = buildreq.Requirements(url)
+        self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf, reqs, content)
+
+        def mock_write(string):
+            self.WRITES.append(string)
+
+        self.specfile._write = mock_write
+        self.specfile._write_strip = mock_write
+        self.WRITES = []
+
+    def test_write_comment_header(self):
+        """
+        test Specfile.write_comment_header with all optional fields
+        """
+        self.specfile.keyid = '1'
+        self.specfile.email = 'email'
+        self.specfile.write_comment_header()
+        expect = ["#\n",
+                  "# This file is auto-generated. DO NOT EDIT\n",
+                  "# Generated by: autospec.py\n",
+                  "# Using build pattern: make\n",
+                  "#\n",
+                  "# Source0 file verified with key 0x1 (email)",
+                  "#"]
+        # skipping autospec version lines as they change
+        self.WRITES = self.WRITES[:4] + self.WRITES[6:]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_nvr_no_urlban(self):
+        """
+        test Specfile.write_nvr with no urlban set
+        """
+        self.specfile.write_nvr()
+        expect = ["Name     : pkg\n",
+                  "Version  : 1.0\n",
+                  "Release  : 2\n",
+                  "URL      : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n",
+                  "Source0  : http://www.testpkg.com/testpkg/pkg-1.0.tar.gz\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_nvr_urlban(self):
+        """
+        test Specfile.write_nvr with urlban set
+        """
+        self.specfile.config.urlban = "www.testpkg.com"
+        self.specfile.write_nvr()
+        expect = ["Name     : pkg\n",
+                  "Version  : 1.0\n",
+                  "Release  : 2\n",
+                  "URL      : http://localhost/testpkg/pkg-1.0.tar.gz\n",
+                  "Source0  : http://localhost/testpkg/pkg-1.0.tar.gz\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_sources(self):
+        """
+        test write_sources with all Specfile.sources set.
+        """
+        self.specfile.config.sources["unit"] = ["pkg2.service", "pkg1.service"]
+        self.specfile.config.sources["archive"] = ["archA", "archD", "archB", "archC"]
+        self.specfile.config.sources["tmpfile"] = ["tmp1", "tmp2"]
+        self.specfile.config.sources["gcov"] = ["pkg.gcov"]
+        self.specfile.write_sources()
+        expect = ["Source1  : archA\n",
+                  "Source2  : archB\n",
+                  "Source3  : archC\n",
+                  "Source4  : archD\n",
+                  "Source5  : pkg.gcov\n",
+                  "Source6  : pkg1.service\n",
+                  "Source7  : pkg2.service\n",
+                  "Source8  : tmp1\n",
+                  "Source9  : tmp2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_summary(self):
+        """
+        test write_summary with unstripped summary and group strings
+        """
+        self.specfile.default_sum = "   This is unstripped summary  "
+        self.specfile.write_summary()
+        expect = ["Summary  : This is unstripped summary\n",
+                  "Group    : Development/Tools\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_license(self):
+        """
+        test write_license with unsorted list of licenses
+        """
+        self.specfile.licenses = ["MIT", "IJG", "GPL-3.0", "GPL-2.0", "ICU"]
+        self.specfile.write_license()
+        expect = ["License  : GPL-2.0 GPL-3.0 ICU IJG MIT\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_main_subpackage_requires_no_autostart(self):
+        """
+        test write_main_subpackage_requires with autostart package and
+        Specfile.no_autostart set.
+        """
+        self.specfile.packages["autostart"] = ["autostart"]
+        self.specfile.packages["bin"] = []
+        self.specfile.packages["lib"] = ["package.so"]
+        self.specfile.requirements.requires[None] = set(["pkg1", "pkg2"])
+        self.specfile.config.config_opts['no_autostart'] = True
+        self.specfile.write_main_subpackage_requires()
+        expect = ["Requires: pkg-bin = %{version}-%{release}\n",
+                  "Requires: pkg-lib = %{version}-%{release}\n",
+                  "Requires: pkg1\n",
+                  "Requires: pkg2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_main_subpackage_requires_autostart(self):
+        """
+        test write_main_subpackage_requires with autostart package and
+        Specfile.no_autostart not set. Also has other ignored packages.
+        """
+        self.specfile.packages["autostart"] = ["autostart"]
+        self.specfile.packages["bin"] = []
+        self.specfile.packages["lib"] = ["package.so"]
+        self.specfile.packages["main"] = ["main/file.py"]
+        self.specfile.packages["ignore"] = []
+        self.specfile.packages["dev"] = []
+        self.specfile.packages["active-units"] = []
+        self.specfile.requirements.requires[None] = set(["pkg1", "pkg2"])
+        self.specfile.write_main_subpackage_requires()
+        expect = ["Requires: pkg-autostart = %{version}-%{release}\n",
+                  "Requires: pkg-bin = %{version}-%{release}\n",
+                  "Requires: pkg-lib = %{version}-%{release}\n",
+                  "Requires: pkg1\n",
+                  "Requires: pkg2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_main_subpackage_requires_empty(self):
+        """
+        test write_main_subpackage_requires with no required subpackages.
+        """
+        self.specfile.write_main_subpackage_requires()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_buildreq(self):
+        """
+        test write_buildreq with unsorted list of build requirements.
+        """
+        self.specfile.requirements.buildreqs = ["python", "ruby", "go"]
+        self.specfile.write_buildreq()
+        expect = ["BuildRequires : go\n",
+                  "BuildRequires : python\n",
+                  "BuildRequires : ruby\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_buildreq_empty(self):
+        """
+        test write_buildreq with no build requirements.
+        """
+        self.specfile.write_buildreq()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_patch_header(self):
+        """
+        test write_patch_header with list of patches.
+        """
+        self.specfile.config.patches = ["speedup.patch",
+                                        "slowdown.patch",
+                                        "revert.patch reverts slowdown.patch"]
+        self.specfile.write_patch_header()
+        expect = ["Patch1: speedup.patch\n",
+                  "Patch2: slowdown.patch\n",
+                  "Patch3: revert.patch\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_patch_header_empty(self):
+        """
+        test write_patch_header with empty patch list.
+        """
+        self.specfile.write_patch_header()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_description(self):
+        """
+        test write_description with unstripped description
+        """
+        self.specfile.default_desc = " This package is for testing only       "
+        self.specfile.write_description()
+        expect = ["\n%description\nThis package is for testing only\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_header_base(self):
+        """
+        base test for write_files_header
+        """
+        self.specfile.packages["data"] = ["df1", "df2", "df3"]
+        self.specfile.packages["ignore"] = ["doesn't matter"]
+        self.specfile.packages["python"] = ["pyfile1", "pyfile2"]
+        self.specfile.packages["dev"] = ["dev1", "dev2"]
+        self.specfile.packages["pack"] = ["packf1"]
+        self.specfile.requirements.buildreqs = ["pep8", "pycurl"]
+        self.specfile.write_files_header()
+        expect = ["\n%package data\n",
+                  "Summary: data components for the pkg package.\n",
+                  "Group: Data\n",
+                  "\n%description data\n",
+                  "data components for the pkg package.\n",
+                  "\n",
+                  "\n%package dev\n",
+                  "Summary: dev components for the pkg package.\n",
+                  "Group: Development\n",
+                  "Requires: pkg-data = %{version}-%{release}\n",
+                  "Provides: pkg-devel = %{version}-%{release}\n",
+                  "Requires: pkg = %{version}-%{release}\n",
+                  "\n%description dev\n",
+                  "dev components for the pkg package.\n",
+                  "\n",
+                  "\n%package pack\n",
+                  "Summary: pack components for the pkg package.\n",
+                  "Group: Default\n",
+                  "\n%description pack\n",
+                  "pack components for the pkg package.\n",
+                  "\n",
+                  "\n%package python\n",
+                  "Summary: python components for the pkg package.\n",
+                  "Group: Default\n",
+                  "\n%description python\n",
+                  "python components for the pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_header_custom_extra_requires(self):
+        """
+        test write_files_header with custom extras requires.
+        """
+        self.specfile.packages["data"] = ["file1"]
+        self.specfile.packages["foobar-extras"] = ["file2"]
+        self.specfile.file_maps = { 'foobar-extras': { 'requires': ["data"] }}
+        self.specfile.write_files_header()
+        expect = ["\n%package data\n",
+                  "Summary: data components for the pkg package.\n",
+                  "Group: Data\n",
+                  "\n%description data\n",
+                  "data components for the pkg package.\n",
+                  "\n",
+                  "\n%package foobar-extras\n",
+                  "Summary: foobar-extras components for the pkg package.\n",
+                  "Group: Default\n",
+                  "Requires: pkg-data = %{version}-%{release}\n",
+                  "\n%description foobar-extras\n",
+                  "foobar-extras components for the pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_header_python_name(self):
+        """
+        test write_files_header with uppercase letter in Specfile.name, causing
+        a Provides line to be written.
+        """
+        self.specfile.name = 'Pkg'
+        self.specfile.packages["python"] = ["pyfile1", "pyfile2"]
+        self.specfile.write_files_header()
+        expect = ["\n%package python\n",
+                  "Summary: python components for the Pkg package.\n",
+                  "Group: Default\n",
+                  "Provides: pkg-python\n",
+                  "\n%description python\n",
+                  "python components for the Pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
+
+    def test_write_files_header_bare(self):
+        """
+        test write_files_header with no packages
+        """
+        self.specfile.write_files_header()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_buildpattern(self):
+        """
+        test write_buildpattern
+        """
+        # Should be doable but probably worth waiting for specfile template rework
+        pass
+
+    def test_write_scriplets_base(self):
+        """
+        test write_scriplets base test
+        """
+        def mock_read_conf_file(name):
+            prefix = "pre" if "pre" in name else "post"
+            return ["{}-script line 1\n".format(prefix),
+                    "{}-script line 2\n".format(prefix),
+                    "{}-script line 3\n".format(prefix)]
+
+        class MockSpecfile(object):
+            @staticmethod
+            def writelines(stringlist):
+                self.WRITES.extend(stringlist)
+
+        self.specfile.config.read_conf_file = mock_read_conf_file
+        self.specfile.specfile = MockSpecfile()
+        self.specfile.packages["ruby"] = ["rubyfile1"]
+        self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
+        self.specfile.write_scriplets()
+        expect = ["\n%post ruby\n",
+                  "post-script line 1\n\n",
+                  "post-script line 2\n\n",
+                  "post-script line 3\n\n",
+                  "\n%pre ruby\n",
+                  "pre-script line 1\n\n",
+                  "pre-script line 2\n\n",
+                  "pre-script line 3\n\n"]
+
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_scriplets_pre_only(self):
+        """
+        test write_scriplets with only pre-scripts present.
+        """
+        def mock_read_conf_file(name):
+            if "pre" in name:
+                return ["pre-script line 1\n",
+                        "pre-script line 2\n",
+                        "pre-script line 3\n"]
+            else:
+                return []
+
+        class MockSpecfile(object):
+            @staticmethod
+            def writelines(stringlist):
+                self.WRITES.extend(stringlist)
+
+        self.specfile.config.read_conf_file = mock_read_conf_file
+        self.specfile.specfile = MockSpecfile()
+        self.specfile.packages["ruby"] = ["rubyfile1"]
+        self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
+        self.specfile.write_scriplets()
+        expect = ["\n%pre ruby\n",
+                  "pre-script line 1\n\n",
+                  "pre-script line 2\n\n",
+                  "pre-script line 3\n\n"]
+
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_scriplets_bare(self):
+        """
+        test write_scriplets with no scripts present.
+        """
+        def mock_read_conf_file(name):
+            return []
+
+        class MockSpecfile(object):
+            @staticmethod
+            def writelines(stringlist):
+                self.WRITES.extend(stringlist)
+
+        self.specfile.config.read_conf_file = mock_read_conf_file
+        self.specfile.specfile = MockSpecfile()
+        self.specfile.packages["ruby"] = ["rubyfile1"]
+        self.specfile.packages["ignore"] = ["ignorefile1", "ignorefile2"]
+        self.specfile.write_scriplets()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_files_base(self):
+        """
+        test write_files base test.
+        """
+        self.specfile.packages["main"] = ["mainfile1", "/mainfile2", "/mainfile3",
+                "/mainfile 4", "mainfile\t5", "%foo /mainfile6", "%bar /mainfile 7"]
+        self.specfile.packages["ignore"] = ["ignorepkg"]
+        self.specfile.packages["other"] = ["other2", "other1"]
+        self.specfile.write_files()
+        # Note the special sorting
+        expect = ["\n%files\n",
+                  "%defattr(-,root,root,-)\n",
+                  '%bar "/mainfile 7"\n',
+                  "%foo /mainfile6\n",
+                  '"/mainfile 4"\n',
+                  "/mainfile2\n",
+                  "/mainfile3\n",
+                  '"mainfile\t5"\n',
+                  "mainfile1\n",
+                  "\n%files other\n",
+                  "%defattr(-,root,root,-)\n",
+                  "other1\n",
+                  "other2\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_files_empty(self):
+        """
+        test write_files with empty package list.
+        """
+        self.specfile.write_files()
+        expect = ["\n%files\n",
+                  "%defattr(-,root,root,-)\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_lang_files_base(self):
+        """
+        write_lang_files base test
+        """
+        self.specfile.locales = ["pkg1", "pkg2"]
+        self.specfile.write_lang_files()
+        expect = ["\n%files locales",
+                  " -f pkg1.lang",
+                  " -f pkg2.lang",
+                  "\n%defattr(-,root,root,-)\n\n"]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_lang_files_empty(self):
+        """
+        test write_lang_files with empty lang list
+        """
+        self.specfile.write_lang_files()
+        self.assertEqual([], self.WRITES)
+
+    def test_write_service_restart(self):
+        """
+        Validate that service_restart configuration is written to the spec file
+        correctly.
+        """
+        self.specfile.config.service_restart = [
+                "/usr/lib/systemd/system/foo.service",
+                "/usr/lib/systemd/system/bar.service",
+        ]
+        self.specfile.write_service_restart()
+        expect = [
+            "## service_restart content",
+            "mkdir -p %{buildroot}/usr/share/clr-service-restart",
+            "ln -s /usr/lib/systemd/system/foo.service %{buildroot}/usr/share/clr-service-restart/foo.service",
+            "ln -s /usr/lib/systemd/system/bar.service %{buildroot}/usr/share/clr-service-restart/bar.service",
+            "## service_restart end",
+        ]
+        self.assertEqual(expect, self.WRITES)
+
+    def test_write_exclude_deletes(self):
+        """
+        Validate that excludes configuration is written to the spec file correctly.
+        """
+        self.specfile.excludes = [
+                "/usr/bin/bar",
+                "/usr/bin/foo",
+        ]
+        self.specfile.write_exclude_deletes()
+        expect = [
+            "## Remove excluded files",
+            "rm -f %{buildroot}*/usr/bin/bar",
+            "rm -f %{buildroot}*/usr/bin/foo",
+        ]
+        self.assertEqual(expect, self.WRITES)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_tarball.py

@@ -0,0 +1,236 @@
+import copy
+import os
+import unittest
+from collections import OrderedDict
+from unittest.mock import MagicMock, Mock, patch
+import build
+import config
+import files
+import tarball
+
+
+# Stores all test cases for dynamic tests.
+# In order to add more tests just add more elements to the lists provided below.
+
+CONTENT_PECL = [
+    'package.xml',
+    'common-prefix/',
+    'common-prefix/md5/',
+    'common-prefix/md5/CMakeLists.txt',
+    'common-prefix/md5/md5.h',
+    'common-prefix/md5/md5hl.c',
+    'common-prefix/md5/md5cmp.c',
+    'common-prefix/md5/md5.c',
+    'common-prefix/md5/Makefile.am',
+    'common-prefix/md5/Makefile.in',
+    'common-prefix/jerror.c',
+    'common-prefix/sharedlib/',
+    'common-prefix/sharedlib/CMakeLists.txt',
+    'common-prefix/turbojpeg-mapfile',
+    'common-prefix/jdpostct.c',
+    'common-prefix/turbojpeg-jni.c',
+]
+
+CONTENT_PREFIX = [
+    'common-prefix/',
+    'common-prefix/md5/',
+    'common-prefix/md5/CMakeLists.txt',
+    'common-prefix/md5/md5.h',
+    'common-prefix/md5/md5hl.c',
+    'common-prefix/md5/md5cmp.c',
+    'common-prefix/md5/md5.c',
+    'common-prefix/md5/Makefile.am',
+    'common-prefix/md5/Makefile.in',
+    'common-prefix/jerror.c',
+    'common-prefix/sharedlib/',
+    'common-prefix/sharedlib/CMakeLists.txt',
+    'common-prefix/turbojpeg-mapfile',
+    'common-prefix/jdpostct.c',
+    'common-prefix/turbojpeg-jni.c',
+]
+
+CONTENT_SUBDIR = [
+    'dir1/',
+    'dir1/md5/',
+    'dir1/md5/CMakeLists.txt',
+    'dir1/md5/md5.h',
+    'dir1/md5/md5hl.c',
+    'dir1/md5/md5cmp.c',
+    'dir1/md5/md5.c',
+    'dir1/md5/Makefile.am',
+    'dir1/md5/Makefile.in',
+    'dir2/',
+    'dir2/jerror.c',
+    'dir2/sharedlib/',
+    'dir2/sharedlib/CMakeLists.txt',
+    'dir2/turbojpeg-mapfile',
+    'dir2/jdpostct.c',
+    'dir2/turbojpeg-jni.c',
+    'file.c'
+]
+
+# Input for tarball.Source class tests.
+# Structure: (url, destination, path, fake-content, source_type, prefix, subddir)
+SRC_CREATION = [
+    ("https://example/src-PECL.tar", "", "/tmp/src-PECL.tar", CONTENT_PECL, "tar", "common-prefix", None),
+    ("https://example/src-non-PECL.tar", "", "/tmp/src-non-PECL.tar", CONTENT_PECL, "tar", "", "src-non-PECL"),
+    ("https://example/src-prefix.zip", "", "/tmp/src-prefix.zip", CONTENT_PREFIX, "zip", "common-prefix", None),
+    ("https://example/src-subdir.zip", "", "/tmp/src-subdir.zip", CONTENT_SUBDIR, "zip", "", "src-subdir"),
+    ("https://example/src-prefix.tar", "", "/tmp/src-prefix.tar", CONTENT_PREFIX, "tar", "common-prefix", None),
+    ("https://example/src-subdir.tar", "", "/tmp/src-subdir.tar", CONTENT_SUBDIR, "tar", "", "src-subdir"),
+    ("https://example/src-no-extractable.tar", ":", "/tmp/src-no-extractable.tar", None, None, None, None),
+]
+
+
+class MockSrcFile():
+    """Mock class for zipfile and tarfile."""
+
+    def __init__(self, path, mode):
+        self.name = path
+        self.mode = mode
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, traceback):
+        return False
+
+    @classmethod
+    def set_content(cls, content):
+        # deep copy because the content is modified by Source
+        cls.content = copy.deepcopy(content)
+
+    def getnames(self):
+        return self.content
+
+    def namelist(self):
+        return self.content
+
+
+def source_test_generator(url, destination, path, content, src_type, prefix, subdir):
+    """Create test for tarball.Source class using generator template."""
+
+    @patch('tarball.tarfile.open', MockSrcFile)
+    @patch('tarball.zipfile.ZipFile', MockSrcFile)
+    @patch('tarball.tarfile.is_tarfile', Mock(return_value=True))
+    @patch('tarball.zipfile.is_zipfile', Mock(return_value=True))
+    def generator(self):
+        """Test template."""
+        # Set fake content
+        MockSrcFile.set_content(content)
+        if os.path.basename(path) in ['src-PECL.tar']:
+            src = tarball.Source(url, destination, path, 'phpize')
+        else:
+            src = tarball.Source(url, destination, path)
+        self.assertEqual(src.type, src_type)
+        self.assertEqual(src.prefix, prefix, f"fail for: {url}")
+        self.assertEqual(src.subdir, subdir)
+
+    return generator
+
+
+def name_and_version_test_generator(url, name, version):
+    """Create test for tarball.name_and_version method."""
+    def generator(self):
+        """Test template."""
+        conf = config.Config('/download/path')
+        conf.parse_config_versions = Mock(return_value={})
+        # Test four different name/version states for tarball.Content, each in
+        # a subtest. Test failures will print these state numbers for easy
+        # identification:
+        # 0 - no state
+        # 1 - name only
+        # 2 - version only
+        # 3 - name and version
+        for state in range(4):
+            with self.subTest(state=state):
+                name_arg = ""
+                version_arg = ""
+                if state == 1 or state == 3:
+                    name_arg = f"state.{name}"
+                if state == 2 or state == 3:
+                    version_arg = f"state.{version}"
+                content = tarball.Content(url, name_arg, version_arg, [], conf, '/tmp')
+                content.config = conf
+                pkg = build.Build()
+                mgr = files.FileManager(conf, pkg)
+                content.name_and_version(mgr)
+                name_cmp = name
+                version_cmp = version
+                if state == 1 or state == 3:
+                    name_cmp = name_arg
+                if state == 2 or state == 3:
+                    version_cmp = version_arg
+                self.assertEqual(name_cmp, content.name)
+                self.assertEqual(version_cmp, content.version)
+        # redo without args and verify giturl is set correctly
+        content.name = ""
+        content.version = ""
+        content.name_and_version(Mock())
+        if "github.com" in url:
+            self.assertRegex(content.giturl, r"https://github.com/[^/]+/" + content.repo + ".git")
+
+    return generator
+
+
+def create_dynamic_tests():
+    """Create dynamic tests based on content in lists and packageulrs file."""
+    # Create tests for tarball.Source class.
+    for url, dest, path, content, src_type, prefix, subdir in SRC_CREATION:
+        test_name = 'test_src_{}'.format(url)
+        test = source_test_generator(url, dest, path, content, src_type, prefix, subdir)
+        setattr(TestTarball, test_name, test)
+
+    # Create tests for tarball.name_and_version method.
+    with open('tests/packageurls', 'r') as pkgurls:
+        for urlline in pkgurls.read().split('\n'):
+            if not urlline or urlline.startswith('#'):
+                continue
+            (url, name, version) = urlline.split(',')
+            test_name = 'test_name_ver_{}'.format(url)
+            test = name_and_version_test_generator(url, name, version)
+            setattr(TestTarball, test_name, test)
+
+
+class TestTarball(unittest.TestCase):
+    """Main testing class for tarball.py.
+
+    This class would contain all static tests and dynamic tests for tarball.py
+    """
+
+    def setUp(self):
+        """Set up default values before start test."""
+        # Set strenght to 0 so it can be updated during tests
+        conf = config.Config('/download/path')
+        self.content = tarball.Content('', '', '', [], conf, '/tmp')
+        conf.content = self.content
+
+    @patch('tarball.os.path.isfile', Mock(return_value=True))
+    def test_set_gcov(self):
+        """Test for tarball.set_gcov method."""
+        # Set up input values
+        self.content.name = 'test'
+        self.content.set_gcov()
+        self.assertEqual(self.content.gcov_file, 'test.gcov')
+
+    @patch('tarball.Source.set_prefix', Mock())
+    @patch('tarball.Source.extract', Mock())
+    def test_extract_sources(self):
+        """Test for Content extract_sources method."""
+        # Set up input values
+        main_src = tarball.Source('https://example1.tar', '', '/tmp')
+        arch1_src = tarball.Source('https://example2.tar', '', '/tmp')
+        arch2_src = tarball.Source('https://example3.tar', ':', '/tmp')
+        arch3_src = tarball.Source('https://example4.tar', '', '/tmp')
+        archives_src = [arch1_src, arch2_src, arch3_src]
+        self.content.extract_sources(main_src, archives_src)
+        # Sources with destination=':' should not be extracted, so method
+        # should be called only 3 times.
+        self.assertEqual(tarball.Source.extract.call_count, 3)
+
+
+# Create dynamic tests based on config file
+create_dynamic_tests()
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/test_util.py

@@ -0,0 +1,68 @@
+import subprocess
+import os
+import tempfile
+import unittest
+import unittest.mock
+import util
+
+
+def mock_gen(rv=None):
+    def mock_f(*args, **kwargs):
+        return rv
+
+    return mock_f
+
+
+class TestUtil(unittest.TestCase):
+
+    def test_call(self):
+        """
+        Test call with default arguments, make sure it passes out the correct
+        returncode
+        """
+        call_backup = subprocess.call
+        util.subprocess.call = mock_gen(rv=0)
+        self.assertEqual(util.call('some command'), 0)
+        util.subprocess.call = call_backup
+
+    def test_call_check(self):
+        """
+        Test call with check=True (default) and a bad returncode. Should raise a
+        CalledProcessError
+        """
+        call_backup = subprocess.call
+        util.subprocess.call = mock_gen(rv=1)
+        with self.assertRaises(subprocess.CalledProcessError):
+            util.call('some command')
+
+        util.subprocess.call = call_backup
+
+    def test_call_no_check(self):
+        """
+        Test call with check=False and a bad returncode, should return the
+        returncode
+        """
+        call_backup = subprocess.call
+        util.subprocess.call = mock_gen(rv=1)
+        self.assertEqual(util.call('some command', check=False), 1)
+        util.subprocess.call = call_backup
+
+    def test_translate(self):
+        """
+        Spot-test the translate function with a package defined in
+        translate.dic
+        """
+        self.assertEqual(util.translate('dateutil-python'), 'pypi-python_dateutil')
+
+    def test_binary_in_path(self):
+        """
+        Test binary_in_path
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            open(os.path.join(tmpd, 'testbin'), 'w').close()
+            util.os.environ["PATH"] = tmpd
+            self.assertTrue(util.binary_in_path('testbin'))
+            self.assertEqual(util.os_paths, [tmpd])
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

tests/testfiles/pkg_integrity/pkg-config-0.29.1.tar.gz.asc

@@ -0,0 +1,17 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJW2ZtDAAoJEAI6RCDH7GkUc0IP/1bH7KEJdUM+lrGM1SOuNHdq
+4VEwDp1II8abbBzHeGEXZ8p4+MwwSOYHFiy+NM1yldZkDXtqAlAqvIuEzc+PtgGd
+vFeNPb9infibNaEDK+zz4fcqJOSab1ZcQ/D3EIJXwKr5nIYP8RuCHu/zstf7o6R0
+/wnGWaAIB1+p9PxvUhMPMbBEQCw/cBzyZ2d7nApHF3b0OH2wM7P8VG8ot4cuglPq
+hzk27ZnrYeUDyUUhMRlL7sZZouJlSy/0OxsBK++tOjE6MiuAZhqtlSW+cFK4L7k/
+q4eLodX7GtF0psSgTRjTk2ozdSIDkB2ccLBN6CzgCcbPrbcz4tVQqaQBcSd0mCl7
+RWAKmSye7p+CY8mIIOjdYm+KaQRmJMKDXs49hMycti22jnu5T2BM6O7MZpiY+cb3
+O2UKUXbVyX/cXKwTYwf4VMddxJKFaqYac+7n5qWbdwBjk9E5OC2ltz94taM1pxZ5
+2jRtfyIb3s+Rj6M5cXI5UChrGqzMK6BmEbyZ0KbHAJ7Y0xvGqwydC6J+RwGIqRlp
+LmW3k1ggpUajoMcgq9KqJgVqo/9f4+6anADHRMNJ93MxR7h5BRQ1/GWSXpOzYsYv
+DnfQPrhc+z20m81qsvfUcBnN/k74yiDVqp3I/HrYGD+f8cXKPpBRESkPAXlUIcu2
+ALHZXEdBGcWdUrAIpJT2
+=SMbV
+-----END PGP SIGNATURE-----

tests/testfiles/pkg_integrity/quagga-1.1.0.tar.gz.asc

@@ -0,0 +1,18 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQJMBAABCAA2BQJYBh/XLxpodHRwczovL3d3dy5qYWttYS5vcmcvfnBhdWwvcGdw
+X3BvbGljeS0xLjEudHh0AAoJEG/lfKjBpK6mNOMP/irlUcU/4NQ6BQQbF7Vc90wo
+hmTH/zGAodnndxSplIGV63BQqmQr5KiSOp6tNQ3OlCIwLFWlr/LiJz+MeThaMtZx
+/mlrZptWIhbEcJzVa6efg8UGIYl2NC+QxFgEyezfCYSEmczd1qE3kiX+5WjcqVPH
+pVjisPwaAYa0FTCsAwBMUKyJr2K3sq5hSJR7DFDVnvH1DJ1xZd9871ZdDpZBHz2z
+GvR+IuZcWbYleH1ArFvT6cMokTpIUMvd33/+Gdpfu9fihzQQn199nN2LFZJzuEGV
+vH6+IZVTmtXb5U/sdtbWGaDv8eFLAWl2NH9VNdlVw5FbWOYRC4YNN39/hBy7s0po
+hvq33ZugC7JqPuje+4W1oF1T/dbRCIBmWUzKsoH8+3z4KJHdotSR0cU+TT+w6SJC
+QhF9TAbjrfbeJs0D0NnZrllDLiLGFgLl5yULzMjRDqKcgqE5+nvYBPXHPUCCPU7+
+59QOkPMsz/kZGV1lRzoUoxlM6V/phJRPU7jit10puiNij0c4peWbjVmTWDei3Xeu
+kHpck5wIAUFzAIXBUpVhYdLl/kxq654Qqthoci869ATZIKH/SOId9Y0K1GE6xO6H
+7vQFUGn3dcdnbgapM+tpmC77EmV4BtzkTjcu2pX6s0qnI4P+M+zXS+7G96xP3zHN
+ZSFq45swEKd6SKbOYxcN
+=mt1F
+-----END PGP SIGNATURE-----