gnulib/pygnulib.py

#!/usr/bin/python
# encoding: UTF-8



import codecs
import collections
import functools
import os
import re
import stat
import sys
import tempfile
import traceback
import subprocess as sp


from pygnulib.error import CommandLineError
from pygnulib.error import UnknownModuleError

from pygnulib.config import BaseConfig
from pygnulib.config import CachedConfig
from pygnulib.config import LGPL_LICENSES
from pygnulib.config import LGPLv3_LICENSE
from pygnulib.config import LGPLv2_LICENSE
from pygnulib.config import GPLv2_LICENSE

from pygnulib.generator import gnulib_cache
from pygnulib.generator import gnulib_comp
from pygnulib.generator import lib_makefile
from pygnulib.generator import po_make_vars
from pygnulib.generator import tests_makefile

from pygnulib.module import DummyModule
from pygnulib.module import Database

from pygnulib.parser import CommandLine as CommandLineParser

from pygnulib.misc import Executable

from pygnulib.vfs import BaseVFS
from pygnulib.vfs import GnulibGitVFS
from pygnulib.vfs import backup as vfs_backup
from pygnulib.vfs import compare as vfs_compare
from pygnulib.vfs import copy as vfs_copy
from pygnulib.vfs import exists as vfs_exists
from pygnulib.vfs import hardlink as vfs_hardlink
from pygnulib.vfs import lookup as vfs_lookup
from pygnulib.vfs import move as vfs_move
from pygnulib.vfs import iostream as vfs_iostream
from pygnulib.vfs import symlink as vfs_symlink
from pygnulib.vfs import unlink as vfs_unlink



class GnulibExecutable(Executable):
    def __init__(self, name, var, prefix, environ=os.environ, encoding=None):
        if not isinstance(var, str):
            raise TypeError("var: str expected")
        if not isinstance(prefix, str):
            raise TypeError("prefix: str expected")
        var = environ.get(var, None)
        prefix = environ.get(prefix, None)
        path = os.path.normpath(name)
        if not var and prefix:
            path = f"{prefix}{path}"
        elif var and not prefix:
            path = os.path.normpath(var)
        super().__init__(path=path, encoding=encoding)



AUTOCONF = GnulibExecutable("autoconf", "AUTOCONF", "AUTOCONFPATH")
AUTOHEADER = GnulibExecutable("autoheader", "AUTOHEADER", "AUTOCONFPATH")
ACLOCAL = GnulibExecutable("aclocal", "ACLOCAL", "AUTOMAKEPATH")
AUTOMAKE = GnulibExecutable("automake", "AUTOMAKE", "AUTOMAKEPATH")
AUTORECONF = GnulibExecutable("autoreconf", "AUTORECONF", "AUTOCONFPATH")
LIBTOOLIZE = GnulibExecutable("libtoolize", "LIBTOOLIZE", "LIBTOOLPATH")
RSYNC = GnulibExecutable("rsync", "RSYNC", "RSYNCPATH")
WGET = GnulibExecutable("wget", "WGET", "WGETPATH")
PATCH = GnulibExecutable("patch", "PATCH", "PATCHPATH")



SUBSTITUTION = {
    "build-aux": "auxdir",
    "doc": "doc_base",
    "lib": "source_base",
    "m4": "m4_base",
    "tests": "tests_base",
    "tests=lib": "tests_base",
    "po": "po_base",
    "top": "",
}
EXECUTABLES = {
    "AUTOCONF": "AUTOCONFPATH",
    "AUTOHEADER": "AUTOCONFPATH",
    "ACLOCAL": "AUTOMAKEPATH",
    "AUTOMAKE": "AUTOMAKEPATH",
    "AUTORECONF": "AUTOCONFPATH",
    "LIBTOOLIZE": "LIBTOOLPATH",
}
for (_var, _prefix) in tuple(EXECUTABLES.items()):
    _name = _var.lower()
    del EXECUTABLES[_var]
    EXECUTABLES[_name] = GnulibExecutable(_name, _var, _prefix)
TP_URL = "http://translationproject.org/latest/"
TP_RSYNC_URI = "translationproject.org::tp/latest/"



def list_hook(gnulib, *args, **kwargs):
    (_, _) = (args, kwargs)
    for module in sorted(gnulib.modules()):
        print(module.name, file=sys.stdout)
    return os.EX_OK



def extract_hook(program, gnulib, mode, namespace, *args, **kwargs):
    (_, _) = (args, kwargs)
    mode = mode.replace("extract-", "")
    mode += "s" if mode == "maintainer" else ""
    for module in namespace["modules"]:
        try:
            module = gnulib.module(module)
            print(module[mode], file=sys.stdout)
        except UnknownModuleError as error:
            print("{0}:".format(program), "warning:", error, file=sys.stderr)
    return os.EX_OK



def import_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs):
    (_, _) = (args, kwargs)
    config = BaseConfig(**namespace)
    cache = CachedConfig(root=config.root, m4_base=config.m4_base)
    database = Database(gnulib.module, config)

    # Print some information about modules.
    print("Module list with included dependencies (indented):", file=sys.stdout)
    BOLD_ON = BOLD_OFF = ""
    mode = os.fstat(sys.stdout.fileno()).st_mode
    if not (stat.S_ISFIFO(mode) or stat.S_ISREG(mode)):
        BOLD_ON = "\033[1m"
        BOLD_OFF = "\033[0m"
    for module in database.final_modules:
        manual = module in database.explicit_modules
        prefix = "  " if manual else "    "
        bold_on = BOLD_ON if manual else ""
        bold_off = BOLD_OFF if manual else ""
        print("{0}{1}{2}{3}".format(prefix, bold_on, module.name, bold_off), file=sys.stdout)
    if verbosity >= 1:
        print("Main module list:", file=sys.stdout)
        for module in database.main_modules:
            if module is not DummyModule():
                print("  {0}".format(module.name), file=sys.stdout)
        if database.main_modules:
            print("", file=sys.stdout)
        print("Tests-related module list:", file=sys.stdout)
        for module in database.test_modules:
            print("  {0}".format(module.name), file=sys.stdout)
        if database.test_modules:
            print("", file=sys.stdout)

    # Determine license incompatibilities, if any.
    incompatibilities = set()
    if config.licenses & LGPL_LICENSES:
        acceptable = {
            "GPLed build tool",
            "public domain",
            "unlimited",
            "unmodifiable license text",
        }
        acceptable |= set(config.licenses)
        for (name, licenses) in ((module.name, module.licenses) for module in database.main_modules):
            if not (acceptable & set(licenses)):
                incompatibilities.add((name, licenses))
    if incompatibilities:
        print("{0}: *** incompatible license on modules:".format(script), file=sys.stderr)
        for (name, licenses) in sorted(incompatibilities):
            print(" " * 16, "{0:50}{1}".format(name, " ".join(sorted(licenses))), file=sys.stderr)
        print("{0}: *** Stop.".format(script), file=sys.stderr)
        return os.EX_DATAERR

    # Show banner notice of every module.
    if verbosity >= -1:
        for module in database.main_modules:
            notice = module.notice
            if notice.strip():
                print("Notice from module {0}:".format(module.name), file=sys.stdout)
                print("\n".join("  " + line for line in notice.splitlines()), file=sys.stdout)

    # Determine the final file list.
    test_files = set()
    main_files = set(database.main_files)
    for file in database.test_files:
        if file.startswith("lib/"):
            file = "tests=lib/{}".format(file[len("lib/"):])
        test_files.add(file)
    files = (main_files | test_files)
    if verbosity >= 0:
        print("File list:", file=sys.stdout)
        for file in sorted(files):
            if file.startswith("tests=lib/"):
                name = file[len("tests=lib/"):]
                src = "lib/" + name
                dst = "tests/" + name
                print("  ", src, " -> ", dst, file=sys.stdout, sep="")
            else:
                print("  ", file, file=sys.stdout, sep="")

    dry_run = options["dry_run"]
    gnulib_copymode = config.copymode
    local_copymode = config.local_copymode

    def transfer_file(local, src_vfs, src_name, dst_vfs, dst_name):
        args = (src_vfs, src_name, dst_vfs, dst_name)
        if src_vfs is None:
            return vfs_copy(*args)
        action = {
            None: vfs_copy,
            "hardlink": vfs_hardlink,
            "symlink": vfs_symlink,
        }[local_copymode if local else gnulib_copymode]
        try:
            return action(*args)
        except OSError as error:
            if error.errno == errno.EXDEV:
                return vfs_copy(*args)
            raise error

    def remove_file(project, file):
        action = ("Removing", "Remove")[dry_run]
        fmt = (action + " file {name} (backup in {name}~)")
        if not dry_run:
            try:
                vfs_backup(project, file)
                vfs_unlink(project, file)
            except FileNotFoundError:
                pass
        print(fmt.format(name=project[file]), file=sys.stdout)

    def update_file(local, src_vfs, src_name, dst_vfs, dst_name, present):
        if not vfs_compare(src_vfs, src_name, dst_vfs, dst_name):
            action = (("Replacing", "Replace"), ("Updating", "Update"))[present][dry_run]
            message = ("(non-gnulib code backed up in {name}~) !!", "(backup in {name}~)")[present]
            fmt = (action + " file {name} " + message)
            if not dry_run:
                vfs_backup(dst_vfs, dst_name)
                transfer_file(local, src_vfs, src_name, dst_vfs, dst_name)
            print(fmt.format(name=dst_vfs[dst_name]), file=sys.stdout)

    def add_file(local, src_vfs, src_name, dst_vfs, dst_name, present):
        action = ("Copying", "Copy")[dry_run]
        fmt = (action + " file {name}")
        if not dry_run:
            transfer_file(local, src_vfs, src_name, dst_vfs, dst_name)
        print(fmt.format(name=dst_vfs[dst_name]), file=sys.stdout)

    # Adjust the VFS mappings.
    overrides = []
    project = BaseVFS(config.root)
    overrides = {BaseVFS(override) for override in config.overrides}
    for (alias, key) in SUBSTITUTION.items():
        path = config[key] if key else "."
        if path is not None:
            project[alias] = path
            for override in overrides:
                override[alias] = path
    gnulib["tests=lib"] = "lib"

    # Determine all relevant file lists.
    old_files = set(cache.files)
    if "m4/gnulib-tool.m4" in project:
        old_files |= set(["m4/gnulib-tool.m4"])
    new_files = (files | set(["m4/gnulib-tool.m4"]))

    # Determine required transformations.
    transformations = []
    for module in database.main_modules:
        if module.name == "config-h":
            # Assume config.h exists, and that -DHAVE_CONFIG_H is omitted.
            def transformation(pattern, path, string):
                if path.startswith("lib") or path.startswith("tests=lib"):
                    return pattern.sub(r"#if 1", string)
                return string
            pattern = re.compile(r"^#ifdef\s+HAVE_CONFIG_H\s*$", re.M)
            transformations.append(functools.partial(transformation, pattern))
            break

    licenses = set(config.licenses)
    if licenses == (GPLv2_LICENSE | LGPLv3_LICENSE):
        def transformation(pattern, path, string):
            repl = (
                "   This program is free software: you can redistribute it and/or",
                "   modify it under the terms of either:",
                "",
                "     * the GNU Lesser General Public License as published by the Free",
                "       Software Foundation; either version 3 of the License, or (at your",
                "       option) any later version.",
                "",
                "   or",
                "",
                "     * the GNU General Public License as published by the Free",
                "       Software Foundation; either version 2 of the License, or (at your",
                "       option) any later version.",
                "",
                "   or both in parallel, as here.",
                "",
                "   This program is distributed in the hope that it will be useful,",
                "   but WITHOUT ANY WARRANTY; without even the implied warranty of",
                "   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the",
                "   GNU General Public License for more details.",
                "",
                "   You should have received a copy of the GNU General Public License",
                "   along with this program.  If not, see <https://www.gnu.org/licenses/>.",
            )
            if path.startswith("lib") or path.startswith("lib=tests"):
                return pattern.sub("\n".join(repl), string)
            return string
        pattern = re.compile((r"^\s*(This program is free software.*?)\s*\*/$"), re.S | re.M)
        transformations.append(functools.partial(transformation, pattern))
    elif licenses == LGPLv3_LICENSE:
        transformations += [
            lambda path, string:
                string.replace("GNU General", "GNU Lesser General")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("General Public License", "Lesser General Public License")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("Lesser Lesser General Public License", "Lesser General Public License")
                if path.startswith("lib") else string,
        ]
    elif licenses == LGPLv2_LICENSE:
        transformations += [
            lambda path, string:
                string.replace("GNU General", "GNU Lesser General")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("General Public License", "Lesser General Public License")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("Lesser Lesser General Public License", "Lesser General Public License")
                if path.startswith("lib") else string,
        ]
        transformation = lambda pattern, path, string: pattern.sub(r"version 2.1\1", string)
        def transformation(pattern, path, string):
            if path.startswith("lib") or path.startswith("lib=tests"):
                return pattern.sub(r"version 2.1\1", string)
            return string
        pattern = re.compile(r"version\s+[23]([\s,])")
        transformations.append(functools.partial(transformation, pattern))
    else:
        transformations += [
            lambda path, string:
                string.replace("GNU Lesser General", "GNU General")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("Lesser General Public License", "General Public License")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("GNU Library General", "GNU General")
                if path.startswith("lib") else string,
            lambda path, string:
                string.replace("Library General Public License", "General Public License")
                if path.startswith("lib") else string,
        ]
        def transformation(pattern, path, string):
            if path.startswith("lib") or path.startswith("lib=tests"):
                return pattern.sub(r"version 3\1", string)
            return string
        pattern = re.compile(r"version\s+2(?:\.1){0,1}([\s,])")
        transformations.append(functools.partial(transformation, pattern))

    if config.copyrights:
        for root in {"build-aux", "tests=lib"}:
            def transformation(root, pattern, path, string):
                if path.startswith(root):
                    string = string.replace("GNU Lesser General", "GNU General")
                    string = string.replace("Lesser General Public License", "General Public License")
                    string = string.replace("GNU Library General", "GNU General")
                    string = string.replace("Library General Public License", "General Public License")
                    string = pattern.sub(r"version 3\1", string)
                return string
            transformations.append(functools.partial(transformation, root, pattern))

    # First the files that are in old_files, but not in new_files.
    # Then the files that are in new_files, but not in old_files.
    # Then the files that are in new_files and in old_files.
    removed_files = old_files.difference(new_files)
    added_files = new_files.difference(old_files)
    kept_files = (old_files & new_files)
    for file in sorted(removed_files):
        remove_file(project, file)
    for (present, files) in ((False, added_files), (True, kept_files)):
        for dst in sorted(files):
            (vfs, src) = vfs_lookup(dst, gnulib, overrides, patch=PATCH)
            action = update_file if vfs_exists(project, dst) else add_file
            with vfs_iostream(vfs, src, "rb", "UTF-8") as istream:
                dst_data = src_data = istream.read()
            for transformation in transformations:
                dst_data = transformation(dst, dst_data)
            temporary = False
            local = (vfs and vfs != gnulib)
            if src_data != dst_data:
                with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as ostream:
                    ostream.write(dst_data)
                src = ostream.name
                temporary = True
                local = False
            action(local, vfs, src, project, dst, present)
            if temporary:
                os.unlink(src)

    mkedits = []
    if config.makefile_name in {None, "Makefile.am"}:
        dirname = os.path.dirname(config.source_base)
        basename = os.path.basename(config.source_base)
        dirname += ("", os.path.sep)[bool(dirname)]
        mkedits.append((dirname, "SUBDIRS", basename))
    if "po_base" in explicit:
        dirname = os.path.dirname(config.po_base)
        basename = os.path.basename(config.po_base)
        dirname = "" if dirname == "." else dirname
        mkedits.append((dirname, "SUBDIRS", basename))
    if config.tests and config.makefile_name in {None, "Makefile.am"}:
        dirname = os.path.dirname(config.tests_base)
        basename = os.path.basename(config.tests_base)
        dirname = "" if dirname == "." else (dirname + os.path.sep)
        mkedits.append((dirname, "SUBDIRS", basename))
    mkedits.append(("", "ACLOCAL_AMFLAGS", "-I {}".format(config.m4_base)))

    # Find the first parent directory of m4_base that contains or will contain a Makefile.am.
    (dir1, dir2) = (config.m4_base, "")
    makefile_name = config.makefile_name
    makefile_name = "Makefile.am" if makefile_name is None else makefile_name
    source_base_makefile = os.path.join(config.source_base, makefile_name)
    tests_base_makefile = os.path.join(config.tests_base, makefile_name)
    while dir1 != "./":
        path = os.path.join(dir1, "Makefile.am")
        if vfs_exists(project, os.path.join(config.root, dir1, "Makefile.am")) \
        or (config.tests and path in (source_base_makefile, tests_base_makefile)):
            break
        dir2 = os.path.join(os.path.basename(dir1), dir2)
        dir1 = os.path.dirname(dir1)
        if not dir1:
            dir1 = "."
    mkedits.append((dir1, "EXTRA_DIST", os.path.join(dir2, "gnulib-cache.m4")))

    # Generate the contents of library makefile.
    path = os.path.join("lib", makefile_name)
    with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
        arguments = {
            "path": project[path],
            "config": config,
            "explicit": explicit,
            "database": database,
            "mkedits": mkedits,
            "testing": False,
        }
        for line in lib_makefile(**arguments, autoconf=AUTOCONF):
            print(line, file=tmp)
    (src, dst) = (tmp.name, path)
    present = vfs_exists(project, dst)
    action = update_file if present else add_file
    action(False, None, src, project, dst, present)
    os.unlink(tmp.name)
    if not present:
        added_files.add(dst)

    # Generate po makefile and directories.
    url = (TP_RSYNC_URI + "gnulib/")
    if "po_base" in explicit:
        for file in ("Makefile.in.in", "remove-potcdate.sin"):
            path = os.path.join("build-aux", "po", file)
            match = tuple(override for override in overrides if file in override)
            override = match[0] if match else gnulib
            (vfs, src) = vfs_lookup(path, gnulib, override, patch=PATCH)
            dst = os.path.join("po", file)
            present = vfs_exists(project, dst)
            action = update_file if present else add_file
            action(bool(match), vfs, src, project, dst, present)
            if not present:
                added_files.add(dst)

        # Create po makefile parameterization, part 1.
        with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
            for line in po_make_vars(config):
                print(line, file=tmp)
        (src, dst) = (tmp.name, "po/Makevars")
        present = vfs_exists(project, dst)
        action = update_file if present else add_file
        action(False, None, src, project, dst, present)
        os.unlink(tmp.name)
        if not present:
            added_files.add(dst)

        # Create po makefile parameterization, part 2.
        with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
            for line in po_make_vars(config):
                print(line, file=tmp)
        (src, dst) = (tmp.name, "po/POTFILESGenerator.in")
        present = vfs_exists(project, dst)
        action = update_file if present else add_file
        action(False, None, src, project, dst, present)
        os.unlink(tmp.name)
        if not present:
            added_files.add(dst)

        po_root = os.path.join(project.absolute, project["po"])
        fmt = ("{} gnulib PO files from " + TP_URL)
        print(fmt.format("Fetching", "Fetch")[dry_run], file=sys.stdout)
        if not dry_run:
            # Prefer rsync over wget if it is available, since it consumes
            # less network bandwidth, due to compression.
            url = (TP_RSYNC_URI + "gnulib/")
            cmdargs = ("rsync", "--delete", "--exclude", "*.s1", "-Lrtz", url, ".")
            with sp.Popen(cmdargs, cwd=po_root, stdout=sp.PIPE, stderr=sp.PIPE) as process:
                (stdout, stderr) = process.communicate()
                stdout = stdout.decode("UTF-8")
                stderr = stderr.decode("UTF-8")
                returncode = process.returncode
                if process.returncode == 0:
                    raise sp.CalledProcessError(returncode, cmdargs, stdout, stderr)
                print(stdout, file=sys.stdout)
                print(stderr, file=sys.stderr)
        # Create po/LINGUAS.
        languages = set()
        for root, _, files in os.walk(po_root):
            languages.update(file[:-3] for file in files if file.endswith(".po"))
        with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
            tmp.write("# Set of available languages.\n")
            for line in sorted(languages):
                print(line, file=tmp)
        (src, dst) = (tmp.name, "po/LINGUAS")
        present = vfs_exists(project, dst)
        action = update_file if present else add_file
        action(False, None, src, project, dst, present)
        os.unlink(tmp.name)
        if not present:
            added_files.add(dst)

    # Create m4/gnulib-cache.m4.
    with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
        for line in gnulib_cache(config, explicit):
            print(line, file=tmp)
    (src, dst) = (tmp.name, "m4/gnulib-cache.m4")
    present = vfs_exists(project, dst)
    action = update_file if present else add_file
    action(False, None, src, project, dst, present)
    os.unlink(tmp.name)
    if not present:
        added_files.add(dst)

    # Create m4/gnulib-comp.m4.
    with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
        arguments = {
            "config": config,
            "explicit": explicit,
            "database": database,
        }
        for line in gnulib_comp(**arguments):
            print(line, file=tmp)
    (src, dst) = (tmp.name, "m4/gnulib-comp.m4")
    present = vfs_exists(project, dst)
    action = update_file if present else add_file
    action(False, None, src, project, dst, present)
    os.unlink(tmp.name)
    if not present:
        added_files.add(dst)

    # Generate the contents of tests makefile.
    if config.tests:
        path = os.path.join(config.tests_base, config.makefile_name)
        with tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) as tmp:
            arguments = {
                "path": path,
                "config": config,
                "explicit": explicit,
                "modules": database.test_modules,
                "mkedits": mkedits,
                "testing": False,
                "libtests": database.libtests,
            }
            for line in tests_makefile(**arguments):
                print(line, file=tmp)
        (src, dst) = (tmp.name, path)
        present = vfs_exists(project, dst)
        action = update_file if present else add_file
        action(False, None, src, project, dst, present)
        os.unlink(tmp.name)
        if not present:
            added_files.add(dst)

    # Generate version control files.
    if config.vc_files:
        table = collections.defaultdict(list)
        for path in added_files:
            (directory, name) = os.path.split(path)
            table[directory].append(["+", name])
        for path in removed_files:
            (directory, name) = os.path.split(path)
            table[directory].append(["-", name])

        # Treat gnulib-comp.m4 like an added file, even if it already existed.
        table["m4"].append(["+", "gnulib-comp.m4"])

        visited = set()
        tmp_table = dict(table)
        for directory in tmp_table:
            if project[directory] in visited:
                del table[directory]
            visited.add(project[directory])
        table = dict(table)
        for directory in sorted(table):
            kinds = []
            kinds += ([], [".gitignore"])[os.path.isdir(os.path.join(project.root, ".git"))]
            kinds += ([], [".cvsignore"])[os.path.isdir(os.path.join(project.root, "CVS"))]
            kinds += ([], [".gitignore"])[os.path.isfile(os.path.join(project.root, project[directory], ".gitignore"))]
            kinds += ([], [".cvsignore"])[os.path.isdir(os.path.join(project.root, project[directory], "CVS"))]
            kinds += ([], [".cvsignore"])[os.path.isfile(os.path.join(project.root, project[directory], ".cvsignore"))]
            for kind in set(kinds):
                anchor = {
                    ".gitignore": "/",
                    ".cvsignore": "",
                }[kind]
                path = os.path.join(directory, kind)
                try:
                    with vfs_iostream(project, path,  "rb", "UTF-8") as stream:
                        origin = [line.strip() for line in stream.readlines()]
                    present = True
                except FileNotFoundError:
                    origin = []
                    present = False
                include = set()
                exclude = set()
                for (action, name) in table[directory]:
                    name = f"{anchor}{name}"
                    already = name in origin
                    if action == "-" and not already:
                        exclude.add(name)
                    elif action == "+" and not already:
                        include.add(name)
                result = []
                for entry in origin:
                    if entry not in exclude:
                        result.append(entry)
                for entry in sorted(include):
                    if entry not in origin:
                        result.append(entry)
                if (set(origin) != set(result)):
                    action = (("Creating", "Create"), ("Updating", "Update"))[present][dry_run]
                    fmt = (action + " {name}" + ("", " (backup in {name}~)")[present])
                    print(fmt.format(name=project[path]), file=sys.stdout)
                    if not dry_run:
                        if present:
                            vfs_backup(project, path)
                        with vfs_iostream(project, path,  "wb", "UTF-8") as stream:
                            for entry in result:
                                print(entry, file=stream)

    print("Finished.", file=sys.stdout)
    print("", file=sys.stdout)

    # Mention the required include directives.
    # First the #include <...> directives without #ifs, sorted for convenience,
    # then the #include "..." directives without #ifs, sorted for convenience,
    # then the #include directives that are surrounded by #ifs. Not sorted.
    print("You may need to add #include directives for the following .h files.", file=sys.stdout)
    table = {"quotes": set(), "angles": set(), "other": list()}
    for module in sorted(set(database.explicit_modules) & set(database.main_modules)):
        directives = "\n".join(module.include_directives)
        if directives.startswith("\""):
            table["quotes"].add(directives)
        elif directives.startswith("<"):
            table["angles"].add(directives)
        else:
            table["other"].append(directives)
    for key in ("angles", "quotes"):
        value = sorted(table[key])
        for item in value:
            print(f"  #include {item}", file=sys.stdout)
    for item in table["other"]:
        for line in item.splitlines():
            print(f"  {line}", file=sys.stdout)

    # Mention the required linker directives.
    lines = []
    for module in database.main_modules:
        lines += module.link_directives
    if lines:
        print("", file=sys.stdout)
        print("You may need to use the following Makefile variables when linking.", file=sys.stdout)
        print("Use them in <program>_LDADD when linking a program, or", file=sys.stdout)
        print("in <library>_a_LDFLAGS or <library>_la_LDFLAGS when linking a library.", file=sys.stdout)
        for line in sorted(set(lines)):
            print(f"  {line}", file=sys.stdout)

    # Mention other necessary actions.
    print("", file=sys.stdout)
    print("Don't forget to", file=sys.stdout)
    if config.makefile_name is None:
        config.makefile_name = "Makefile.am"
    if config.makefile_name == "Makefile.am":
        fmt = "  - add \"{source_base}/Makefile\" to AC_CONFIG_FILES in {ac_file},"
    else:
        fmt = "  - \"include {makefile_name}\" from within \"{source_base}/Makefile.am\","
    print(fmt.format(**config), file=sys.stdout)
    if config.po_base:
        fmt = "  - add \"{po_base}/Makefile.in\" to AC_CONFIG_FILES in {ac_file},"
        print(fmt.format(**config), file=sys.stdout)
    if config.tests:
        if config.makefile_name == "Makefile.am":
            fmt = "  - add \"{tests_base}/Makefile\" to AC_CONFIG_FILES in {ac_file},"
        else:
            fmt = "  - \"include {makefile_name}\" from within \"{tests_base}/Makefile.am\","
        print(fmt.format(**config), file=sys.stdout)
    for (directory, key, value) in mkedits:
        path = os.path.normpath(os.path.join(directory, "Makefile.am"))
        print(f"  - mention \"{value}\" in {key} in {path},", file=sys.stdout)
    position_early_after = "AC_PROG_CC"
    with vfs_iostream(project, config.ac_file, "rb", "UTF-8") as stream:
        contents = stream.read()
        AC_PROG_CC_STDC = re.compile(r"^\s*AC_PROG_CC_STDC", re.S | re.M)
        AC_PROG_CC_C99 = re.compile(r"^\s*AC_PROG_CC_C99", re.S | re.M)
        if AC_PROG_CC_STDC.findall(contents):
            position_early_after = "AC_PROG_CC_STDC"
        elif AC_PROG_CC_C99.findall(contents):
            position_early_after = "AC_PROG_CC_C99"
    fmt = "  - invoke {macro_prefix}_EARLY in {ac_file}, right after {position_early_after},"
    print(fmt.format(**config, position_early_after=position_early_after), file=sys.stdout)
    fmt = "  - invoke {macro_prefix}_INIT in {ac_file}."
    print(fmt.format(**config), file=sys.stdout)
    return os.EX_OK



def add_import_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs):
    (_, _) = (args, kwargs)
    modules = set(namespace.pop("modules"))
    config = CachedConfig(**namespace)
    namespace = {k:v for (k, v) in config.items()}
    namespace["modules"] = (config.modules | modules)
    return import_hook(script, gnulib, namespace, verbosity, options)



def remove_import_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs):
    (_, _) = (args, kwargs)
    modules = set(namespace.pop("modules"))
    config = CachedConfig(**namespace)
    namespace = {k:v for (k, v) in config.items()}
    namespace["modules"] = (config.modules - modules)
    return import_hook(script, gnulib, namespace, verbosity, options)



def update_hook(script, gnulib, namespace, explicit, verbosity, options, *args, **kwargs):
    (_, _) = (args, kwargs)
    config = CachedConfig(**namespace)
    namespace = {k:v for (k, v) in config.items()}
    return import_hook(script, gnulib, namespace, verbosity, options)



HOOKS = {
    "list": list_hook,
    "extract": extract_hook,
    "import": import_hook,
    "add-import": add_import_hook,
    "remove-import": remove_import_hook,
    "update": update_hook,
}



def main(script, gnulib, program, arguments, environ):
    gnulib = GnulibGitVFS(gnulib)
    parser = CommandLineParser(program)
    try:
        (namespace, mode, verbosity, options) = parser.parse(arguments)
    except CommandLineError as error:
        print(parser.usage, file=sys.stderr)
        print(error, file=sys.stderr)
        return os.EX_USAGE
    if mode == "help":
        print(parser.help, file=sys.stdout)
        return os.EX_OK
    kwargs = {
        "script": script,
        "program": program,
        "gnulib": gnulib,
        "mode": mode,
        "namespace": namespace,
        "explicit": namespace.keys(),
        "verbosity": verbosity,
        "options": options,
    }
    for (action, hook) in HOOKS.items():
        if mode.startswith(action):
            return hook(**kwargs)
    return os.EX_SOFTWARE



if __name__ == "__main__":
    script = sys.argv[0]
    gnulib = os.path.dirname(__file__)
    program = os.path.basename(script)
    log = os.path.join(os.getcwd(), "{0}.log".format(program))
    arguments = list(sys.argv[1:])
    environ = dict(os.environ)
    try:
        result = main(script, gnulib, program, arguments, environ)
    except StopIteration as error:
        with codecs.open(log, "wb", "UTF-8") as stream:
            program = repr(program) if " " in program else program
            arguments = " ".join(repr(arg) if " " in arg else arg for arg in arguments)
            print(traceback.format_exc(), file=stream)
            print("COMMAND:", program, arguments, file=stream)
        typeid = type(error)
        module = typeid.__module__
        name = typeid.__name__
        if module != "builtins":
            error = "{0}.{1}: {2}".format(module, name, error)
        else:
            error = "{0}: {1}".format(name, error)
        print("{0}:".format(program), "***", error, file=sys.stderr)
        print("{0}:".format(program), "***", log, file=sys.stderr)
        result = os.EX_SOFTWARE
    sys.exit(result)