Update packages file for version 30400

Signed-off-by: clrbuilder <clrbuilder@intel.com>

Makefile.common

@@ -70,10 +70,9 @@ proper: repodel
 mockclean:
 	$(MOCK) --clean --scrub=all
 
-#help mockproper: Clean mock chroot and cache directories for this package.
+#help mockproper: Clean mock chroot
 mockproper:
-	$(MOCK) --clean --scrub=all --uniqueext=$(PKG_NAME)
-	$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
+	$(MOCK) --clean --uniqueext=$(PKG_NAME)
 
 configemail:
 	@test -d .git || git init
@@ -187,6 +186,19 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		bash update_changelog.sh ; \
 		git commit --amend --no-edit Change* ; \
 	fi
+	@$(MAKE) -s for-review.txt
+	diffstat for-review.txt
+	@printf "\n**\n"
+	@printf "** NOTICE: A patch with changes is available in the file for-review.txt\n"
+	@printf "** To recreate (e.g., after git commit --amend), run make for-review.txt\n"
+	@printf "** To submit for review: git send-email --to <recipient> for-review.txt\n"
+	@printf "**\n\n"
+
+#help for-review.txt: Creates the for-review.txt file, which is
+#help a minimized version of the Git commit, suitable for code review.
+for-review.txt:
+	git diff | grep -q index || python3 $(TOPLVL)/projects/common/patchfilter.py > for-review.txt
+.PHONY: for-review.txt
 
 #help autospecnogit: Runs autospec, but does not create a commit
 autospecnogit:
@@ -268,6 +280,7 @@ sources:
 	@$(MAKE) generateupstream; \
 	[ $$? -eq 0 ] && exit 0; \
 	nvr="$$(rpmspec --srpm -q --qf '%{NAME}\t%{VERSION}\t%{RELEASE}\n' $(SPECFILE))"; \
+	git -C $(TOPLVL)/projects/common fetch --tags >/dev/null 2>&1; \
 	latest_builds=$$(git -C $(TOPLVL)/projects/common tag -l | sort -rn); \
 	for b in $$latest_builds; do \
 		url="$(DOWNLOAD_MIRROR)/releases/$$b/clear/source/package-sources"; \
@@ -319,7 +332,7 @@ scratch: $(SPECFILE) upstream $(SRPMFILE) kojidef
 
 #help scratch-wait: Performs scratch-build on package.
 scratch-wait: $(SPECFILE) upstream $(SRPMFILE) kojidef
-	$(KOJI_CMD) build --scratch $(KOJI_TAG) $(SRPMFILE)
+	$(KOJI_CMD) build --scratch --wait $(KOJI_TAG) $(SRPMFILE)
 
 #help koji-nowait: Same as 'make koji', but do not block
 koji-nowait:
@@ -412,27 +425,43 @@ install:
 #help install-debuginfo-local: Install locally built debuginfo RPM to
 #help the automatic debuginfo cache location (/var/cache/debuginfo)
 install-debuginfo-local:
+	@if [ -z "${DEBUGINFO_RPM}" ]; then \
+		echo "No debuginfo to install... skipping"; \
+		exit 0; \
+	fi; \
+	echo -n "Installing ${DEBUGINFO_RPM}... "; \
 	tmpdir=$$(mktemp -d); \
-	rpm2cpio ${DEBUGINFO_RPM} | ( cd $$tmpdir; cpio -i -d -u); \
+	rpm2cpio ${DEBUGINFO_RPM} | ( cd $$tmpdir; cpio -i -d -u 2> /dev/null); \
 	dest=/var/cache/debuginfo/lib; \
-	find $$tmpdir/usr/lib/debug/ -mindepth 1 -maxdepth 1 | while read -r d; do \
+	find $$tmpdir/usr/lib/debug/ -mindepth 1 -maxdepth 1 2> /dev/null | while read -r d; do \
+		sudo chown -R dbginfo:dbginfo "$$d"; \
+		sudo cp -a "$$d" $$dest/; \
+	done; \
+	find $$tmpdir/usr/share/debug/ -mindepth 1 -maxdepth 1 -regextype awk -regex '.*/(.build-id|boot|lib|sbin|usr)$$' 2> /dev/null | while read -r d; do \
 		sudo chown -R dbginfo:dbginfo "$$d"; \
 		sudo cp -a "$$d" $$dest/; \
 	done; \
 	dest=/var/cache/debuginfo/src; \
-	find $$tmpdir/usr/src/debug/ -mindepth 1 -maxdepth 1 | while read -r d; do \
+	find $$tmpdir/usr/src/debug/ -mindepth 1 -maxdepth 1 2> /dev/null | while read -r d; do \
 		sudo chown -R dbginfo:dbginfo "$$d"; \
 		sudo cp -a "$$d" $$dest/; \
 	done; \
-	sudo rm -rf $$tmpdir
+	find $$tmpdir/usr/share/debug/src/ -mindepth 1 -maxdepth 1 2> /dev/null | while read -r d; do \
+		sudo chown -R dbginfo:dbginfo "$$d"; \
+		sudo cp -a "$$d" $$dest/; \
+	done; \
+	sudo rm -rf $$tmpdir; \
+	echo "done"
 
 #help install-local: Install locally built RPMs to the root filesystem. Note that the
 #help debuginfo RPM installs to /var/cache/debuginfo
 install-local:
-	for r in $(filter-out ${DEBUGINFO_RPM},${RPMS}); do \
-		rpm2cpio $$r | (cd /; sudo cpio -i -d -u); \
+	@for r in $(filter-out ${DEBUGINFO_RPM},${RPMS}); do \
+		echo -n "Installing $$r... "; \
+		rpm2cpio $$r | (cd /; sudo cpio -i -d -u 2> /dev/null); \
+		echo "done"; \
 	done
-	$(MAKE) install-debuginfo-local
+	@$(MAKE) -s install-debuginfo-local
 
 #help install-mock: Install locally built RPMs to the mock rootcache
 #help filesystem. This command is usually used with "make shell".
@@ -475,6 +504,9 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+# Define LTS-specific targets in a separate makefile
+-include $(TOPLVL)/projects/common/Makefile.common.lts
+
 # Define site local common targets in a separate makefile
 -include $(TOPLVL)/projects/common/Makefile.common.site_local
 

Makefile.common.lts

@@ -0,0 +1,90 @@
+#-*-makefile-*-
+
+LTSUTILS = python $(TOPLVL)/projects/common/lts/main.py $(PKG_NAME)
+
+#help lts-show: Display a summary of active LTS branches.
+lts-show:
+	@$(LTSUTILS) sanity-check
+	@while read b; do \
+		if git show-ref $$b > /dev/null; then \
+			echo $$b $$(git log --oneline -1 $$b); \
+		else \
+			echo $$b Not found; \
+		fi; \
+	done < $(TOPLVL)/projects/common/lts/active-branches
+
+#help lts-backport: Fast-forward the previous active branch to the current
+#help branch.
+lts-backport:
+	@$(LTSUTILS) sanity-check
+	@newer=$$(git symbolic-ref HEAD); \
+	newer=$${newer#refs/heads/}; \
+	if ! $(LTSUTILS) prev-branch --checkout; then \
+		echo Could not check out previous active branch.; \
+		exit 0; \
+	fi; \
+	if $(LTSUTILS) is-same-version $$newer; then \
+		$(LTSUTILS) fast-forward $$newer; \
+	else \
+		echo Most likely a patch needs to be manually re-applied for this version.; \
+		echo Alternatively, use \"git merge --ff-only $$newer\" to upgrade the package version.; \
+	fi
+
+#help lts-build: Build RPM in Koji, or reuse existing build from older LTS
+#help branch.
+lts-build:
+	@$(LTSUTILS) sanity-check
+	@tag=$$(git tag --points-at); \
+	if [[ -z $$tag ]]; then \
+		echo This commit does not have a tag.; \
+		echo Proceeding to run \"make lts-koji\".; \
+		$(LTSUTILS) prompt || exit; \
+		$(MAKE) lts-koji; \
+		exit; \
+	fi; \
+	older=$$($(LTSUTILS) prev-branch); \
+	if [[ -z $$older ]]; then \
+		echo Could not determine previous active branch.; \
+		exit 1; \
+	fi; \
+	current=$$($(LTSUTILS) current-branch); \
+	if [[ $$(git rev-parse $$older) != $$(git rev-parse $$current) ]]; then \
+		echo Previous branch and current branch have different commits.; \
+		for b in $$older $$current; do echo $$b $$(git log --oneline -1 $$b); done; \
+		echo Proceeding to run \"make lts-koji\".; \
+		$(LTSUTILS) prompt || exit; \
+		$(MAKE) lts-koji; \
+		exit; \
+	fi; \
+	if $(LTSUTILS) can-reuse-binary $$older; then \
+		echo Reusing binary from $$older.; \
+		$(KOJI_CMD) tag-build $$current $(SRPMVERS); \
+	else \
+		echo Could not reuse binary from $$older.; \
+		echo Proceeding to run \"make bump lts-koji\".; \
+		$(LTSUTILS) prompt || exit; \
+		$(MAKE) bump lts-koji; \
+	fi
+	@$(LTSUTILS) next-branch --checkout
+
+#help lts-koji: Same as "koji" but for working on LTS branches. Should not
+#help be called directly, use lts-build instead.
+lts-koji: prekoji-checks kojidef
+	@$(LTSUTILS) sanity-check
+	@$(MAKE) spdxcheck
+	@$(MAKE) checkblacklist
+	@current=$$($(LTSUTILS) current-branch); \
+	if ! grep $$current $(TOPLVL)/projects/common/lts/active-branches; then \
+		echo "Error: Must be on an active branch to submit to koji" >&2; \
+		exit 1; \
+	fi; \
+	if ! git diff --quiet HEAD ${SPECFILE}; then \
+		echo "Error: All changes to ${SPECFILE} must be committed first" >&2; \
+		exit 1; \
+	fi; \
+	if git rev-parse --verify --quiet origin/$$current > /dev/null; then \
+		git pull --rebase; \
+	fi
+	git tag $(SRPMVERS)-lts
+	git push origin $$($(LTSUTILS) current-branch) refs/tags/$(SRPMVERS)-lts
+	$(KOJI_CMD) build $$KOJI_NOWAIT $$($(LTSUTILS) current-branch) $(PKG_BASE_URL)/$(PKG_NAME)?#$(SRPMVERS)-lts

Makefile.toplevel

@@ -11,6 +11,7 @@ SHELL   = /bin/bash
 
 PKGS    = $(shell cat $(PACKAGES_FILE))
 PKGS_LOCAL = $(sort $(filter ${PKGS},$(subst packages/,,$(wildcard packages/*))))
+PKGS_ORPHANED = $(sort $(filter-out common ${PKGS},$(subst packages/,,$(wildcard packages/*))))
 
 proj_PKGS = $(addprefix proj_,$(PROJ_PKGS))
 clone_PKGS = $(addprefix clone_,$(PKGS))
@@ -128,11 +129,9 @@ pull: ${PULL_PKGS}
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
 #help removes the associated package repos from the ./packages tree
 clean-pkgs-dir: $(PACKAGES_FILE)
-	@for p in `ls packages`; do \
-		if ! grep -wq "^$$p$$" "$<" && [ "$$p" != common ]; then \
-			echo "Removing $$p from packages, it is no longer in common/packages."; \
-			rm -rf packages/$$p; \
-		fi \
+	@for p in ${PKGS_ORPHANED}; do \
+		echo "Removing $$p from packages, it is no longer in common/packages."; \
+		rm -rf packages/$$p; \
 	done
 	@echo "packages directory cleaned";
 
@@ -226,8 +225,15 @@ autospecnew: preautospecnew-checks localreponotice
 			$${SETVERSION:+ --version $${SETVERSION}} \
 			${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
 			$(URL); \
-		[ $$? -eq 0 ] && $(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
-		$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
+		if [ $$? -eq 0 ]; then \
+			$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
+			$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
+			python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) format-patch -1 --stdout) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
+			printf "\n**\n"; \
+			printf "** NOTICE: A patch with changes is available in the file $(TOPLVL)/packages/$(NAME)/for-review.txt\n"; \
+			printf "** Please look through this file and if you are unsure, please submit for code review with git send-email\n"; \
+			printf "**\n\n"; \
+		fi; \
 	else \
 		echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
 		exit 1; \

licenses-spdx

@@ -99,6 +99,8 @@ CECILL-2.0
 CECILL-2.1
 CECILL-B
 CECILL-C
+CERN-OHL-1.1
+CERN-OHL-1.2
 CNRI-Jython
 CNRI-Python
 CNRI-Python-GPL-Compatible
@@ -183,6 +185,7 @@ Info-ZIP
 Intel
 Intel-ACPI
 Interbase-1.0
+JPNIC
 JSON
 JasPer-2.0
 LAL-1.2
@@ -325,6 +328,7 @@ Spencer-94
 Spencer-99
 StandardML-NJ
 SugarCRM-1.1.3
+TAPR-OHL-1.0
 TCL
 TCP-wrappers
 TMate
@@ -373,6 +377,7 @@ eGenix
 gSOAP-1.3b
 gnuplot
 iMatix
+libpng-2.0
 libtiff
 mpich2
 psfrag

lts/Makefile

@@ -0,0 +1,5 @@
+TEST = test.test_ltsutils
+
+.PHONY: test
+test:
+	PYTHONPATH=. python -m unittest -v -k .Test $(TEST)

lts/README.md

@@ -0,0 +1,19 @@
+# LTS package maintenance utility
+
+This tooling is designed to automate 2 main tasks that are part of
+the package maintenance workflow of Clear Linux LTS. These tasks are:
+- Back-porting of a patch (e.g. security fix) to older branches.
+- Building RPMs with the intent of sharing binaries of older LTS branches to
+  newer branches whenever possible.
+
+There should be no need to run this tool directly. Instead use the following
+targets defined in Makefile.common.lts:
+- lts-show: Show a summary of active LTS branches
+- lts-backport: Attempt to fast-forward the previous active branch to the current branch
+- lts-build: Build RPM in Koji, or reuse existing build from older branch
+
+"Active" branches correspond to LTS releases that currently have support.
+They are listed in a flat file "active-branches" in "lts" directory, from
+oldest to newest. New entries are added by Clear Linux LTS developers as
+new releases become available, and entries removed as releases become
+obsolete.

lts/ltsutils/package_repo.py

@@ -0,0 +1,52 @@
+import pathlib
+from subprocess import PIPE, CalledProcessError
+from .shell import Shell
+
+class PackageRepo:
+    '''Represents a package repository. Most methods are wrappers of git commands.'''
+    class UnknownCurrentBranchException(Exception): pass
+    class InvalidBranchException(Exception): pass
+
+    def __init__(self, name, path):
+        self.name = name
+        self.path = pathlib.Path(path)
+        self.sh = Shell(self.path)
+
+    def getNVR(self, commit='HEAD'):
+        with self.sh.popen(['git', 'show', '{}:{}.spec'.format(commit, self.name)], stdout=PIPE) as specfile:
+            nvr = self.sh.run('rpmspec --srpm -q --queryformat %{NVR} /dev/stdin', stdin=specfile.stdout)
+        return tuple(nvr.stdout.strip().rsplit('-', maxsplit=2))
+
+    def checkoutBranch(self, branch, allow_remote=False):
+        # allow_remote=True allows checking out a new remote-tracking branch
+        if not allow_remote and not self.hasBranch(branch):
+            raise self.InvalidBranchException(branch)
+        self.sh.run_args(['git', 'checkout', branch], capture_output=False)
+
+    def fastForwardBranch(self, old, new):
+        self.checkoutBranch(old)
+        if not self.hasBranch(new):
+            raise self.InvalidBranchException(new)
+        self.sh.run_args(['git', 'merge', '--ff-only', new], capture_output=False)
+
+    def getActiveBranches(self):
+        toplvl = pathlib.Path(self.path) / '../..'
+        common = toplvl / 'projects/common'
+        active_branches = common / 'lts/active-branches'
+        with active_branches.open() as f:
+            return [line.rstrip() for line in f]
+
+    def getCurrentBranch(self):
+        try:
+            head = self.sh.run('git symbolic-ref HEAD').stdout.strip()
+        except CalledProcessError:
+            raise self.UnknownCurrentBranchException
+
+        refs_heads = 'refs/heads/'
+        assert head.startswith(refs_heads)
+        head = head[len(refs_heads):]
+        return head
+
+    def hasBranch(self, branch):
+        p = self.sh.run_args(['git', 'rev-parse', 'refs/heads/'+branch], check=False)
+        return p.returncode == 0

lts/ltsutils/shell.py

@@ -0,0 +1,27 @@
+import subprocess, shlex
+
+class Shell:
+    # Default options passed to subprocess.run. May be customized per-instance.
+    cwd = None
+    check = True
+    capture_output = True
+    text = True
+
+    def __init__(self, cwd=None):
+        if cwd: self.cwd = cwd
+
+    def run_args(self, args, **kwargs):
+        kwargs1 = {
+                'check': self.check,
+                'capture_output': self.capture_output,
+                'text': self.text,
+                'cwd': self.cwd
+                }
+        kwargs1.update(kwargs)
+        return subprocess.run(args, **kwargs1)
+
+    def run(self, cmd, **kwargs):
+        return self.run_args(shlex.split(cmd), **kwargs)
+
+    def popen(self, args, **kwargs):
+        return subprocess.Popen(args, cwd=self.cwd, **kwargs)

lts/main.py

@@ -0,0 +1,161 @@
+#!/usr/bin/python
+import sys,argparse
+
+from ltsutils.package_repo import PackageRepo
+
+def log(msg, **kwargs):
+    print(msg, file=sys.stderr)
+
+def init_parser():
+    main = argparse.ArgumentParser()
+    main.add_argument('package_name', nargs=1)
+    subparsers = main.add_subparsers(dest='command', metavar='command', required=True)
+
+    # Package maintenance commands
+    p = subparsers.add_parser('prev-branch',
+            help='show previous branch')
+    p.add_argument('--checkout', action='store_true', help='checkout the branch')
+    p = subparsers.add_parser('next-branch',
+            help='show next branch')
+    p.add_argument('--checkout', action='store_true', help='checkout the branch')
+    p = subparsers.add_parser('current-branch',
+            help='show current branch')
+    p = subparsers.add_parser('is-same-version',
+            help='return true if package version is the same as the given branch')
+    p.add_argument('branch', nargs=1)
+    p = subparsers.add_parser('fast-forward',
+            help='fast-forward current branch to a newer branch')
+    p.add_argument('branch', nargs=1)
+
+    # RPM build commands
+    p = subparsers.add_parser('can-reuse-binary',
+            help='check if binary from another branch can be used in current branch')
+    p.add_argument('branch', nargs=1)
+
+    # Other commands
+    p = subparsers.add_parser('prompt',
+            help='prompt user to continue and return appropriate exit code')
+    p = subparsers.add_parser('sanity-check',
+            help='run sanity checks for data consistency')
+
+    return main
+
+def prev_branch(args, repo):
+    active_branches = repo.getActiveBranches()
+    current = repo.getCurrentBranch()
+
+    i = active_branches.index(current)
+    if i == 0:
+        log('Already on oldest active branch.')
+        return False
+
+    prev = active_branches[i-1]
+    print(prev)
+    if args.checkout:
+        repo.checkoutBranch(prev, allow_remote=True)
+
+def next_branch(args, repo):
+    active_branches = repo.getActiveBranches()
+    current = repo.getCurrentBranch()
+
+    i = active_branches.index(current)
+    if i == len(active_branches)-1:
+        log('Already on newest active branch.')
+        return False
+
+    next_ = active_branches[i+1]
+    print(next_)
+    if args.checkout:
+        repo.checkoutBranch(next_, allow_remote=False)
+
+def current_branch(args, repo):
+    current = repo.getCurrentBranch()
+    print(current)
+
+def is_same_version(args, other):
+    current = repo.getCurrentBranch()
+    other = args.branch[0]
+    assert repo.hasBranch(other), 'Branch %s not found' % other
+
+    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (current, other)]
+    if v1 != v2:
+        log('Current version {} does not match version {} on branch {}'.format(v1, v2, other))
+    return v1 == v2
+
+def fast_forward(args, repo):
+    current = repo.getCurrentBranch()
+    newer = args.branch[0]
+
+    log('Fast-forwarding {} to {}'.format(current, newer))
+    repo.fastForwardBranch(current, newer)
+
+def sanity_check(args, repo):
+    ok = True
+    # HEAD must point to a branch
+    try:
+        current = repo.getCurrentBranch()
+    except PackageRepo.UnknownCurrentBranchException:
+        log('Unknown current branch. Has a branch been checked out?')
+        current = None
+        ok = False
+
+    # active-branches file must not be empty
+    active_branches = repo.getActiveBranches()
+    if not len(active_branches):
+        log('No active branches defined. Is active-branches file empty?')
+        ok = False
+    # current branch must be an active branch
+    elif current and current not in active_branches:
+        log('%s is not an active LTS branch.' % current)
+        ok = False
+
+    return ok
+
+def can_reuse_binary(args, repo):
+    # Just compare versions for now
+    # TODO: ABI compatibility testing
+    current = repo.getCurrentBranch()
+    older = args.branch[0]
+    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (older, current)]
+    return v1 == v2
+
+def prompt(args, repo):
+    import selectors
+    timeout = 60
+
+    while True:
+        print('Continue? (y/N): ', end='', flush=True)
+        with selectors.DefaultSelector() as sel:
+            sel.register(sys.stdin, selectors.EVENT_READ)
+            events = sel.select(timeout)
+        if not len(events):
+            print('Timed out after {}s.'.format(timeout))
+            return False
+        else:
+            s = sys.stdin.readline().rstrip('\n')
+            if s in ('Y', 'y', 'N', 'n', ''):
+                break
+
+    if s in ('Y', 'y'):
+        return True
+    else:
+        print('Cancelled.')
+        return False
+
+if __name__=='__main__':
+    args = init_parser().parse_args()
+    repo = PackageRepo(args.package_name[0], '.')
+
+    commands = {
+            'prev-branch': prev_branch,
+            'next-branch': next_branch,
+            'current-branch': current_branch,
+            'is-same-version': is_same_version,
+            'fast-forward': fast_forward,
+            'can-reuse-binary': can_reuse_binary,
+            'prompt': prompt,
+            'sanity-check': sanity_check,
+            }
+    ret = commands[args.command](args, repo)
+    if ret is not None:
+        exit(0 if ret else 1)

lts/test/test_ltsutils.py

@@ -0,0 +1,97 @@
+import unittest
+import os, pathlib, tempfile, logging
+import subprocess, functools
+
+from ltsutils.package_repo import PackageRepo
+import ltsutils.shell
+
+run = functools.partial(subprocess.run, check=True)
+
+class Package:
+    def __init__(self, name):
+        self.name = name
+        self.repo_url = 'https://github.com/clearlinux-pkgs/{}.git'.format(name)
+
+class LTSUtilsTestCase(unittest.TestCase):
+    toplvl = pathlib.Path('../../..')
+    packages = toplvl / 'packages'
+
+    def cloneOrExtractRepo(self):
+        tmpdir = pathlib.Path('/var/tmp/common-lts-test')
+        tmpdir.mkdir(mode=0o700, exist_ok=True)
+        tarball = tmpdir / '{}.tar.gz'.format(self.package.name)
+        if tarball.exists():
+            run(['tar', 'xf', tarball, '-C', self.workdir])
+        else:
+            run(['git', 'clone', self.package.repo_url, self.workdir])
+            run(['tar', 'czf', tarball, '-C', self.workdir, '.'])
+
+    def setUp(self):
+        self._tmpdir = tempfile.TemporaryDirectory(prefix='test-{}-'.format(self.package.name), dir=self.packages)
+        self.workdir = pathlib.Path(self._tmpdir.name)
+        self.cloneOrExtractRepo()
+        self.repo = PackageRepo(self.package.name, self.workdir)
+
+        self._sh = ltsutils.shell.Shell(self.workdir)
+        self._sh.capture_output = False
+
+    def sh(self, cmd):
+        return self._sh.run(cmd)
+
+    def sh_stdout(self, cmd):
+        return self._sh.run(cmd, capture_output=True).stdout.rstrip()
+
+    def tearDown(self):
+        self._tmpdir.cleanup()
+
+class PackageRepoTestCase(LTSUtilsTestCase):
+    def setUp(self):
+        super().setUp()
+        self.L1 = self.package.L1
+        self.L2 = self.package.L2
+        self.sh('git branch L1 %s' % self.L1)
+        self.sh('git branch L2 %s' % self.L2)
+
+    def testGetNVR(self):
+        raise NotImplementedError
+
+    def testHasBranch(self):
+        self.assertTrue(self.repo.hasBranch('L2'))
+        self.assertFalse(self.repo.hasBranch('L3'))
+
+    def testCheckoutBranch(self):
+        self.repo.checkoutBranch('L2')
+        self.assertEqual(self.sh_stdout('git rev-parse HEAD'), self.L2)
+
+        self.assertRaises(PackageRepo.InvalidBranchException, self.repo.checkoutBranch, 'L3')
+
+    def testFastForward(self):
+        self.repo.fastForwardBranch('L1', 'L2')
+        self.assertEqual(self.sh_stdout('git rev-parse L1'), self.L2)
+        self.assertEqual(self.sh_stdout('git rev-parse L2'), self.L2)
+
+    def testGetCurrentBranch(self):
+        self.repo.checkoutBranch('L2')
+        b = self.repo.getCurrentBranch()
+        self.assertEqual(b, 'L2')
+
+        self.sh('git checkout --detach L2')
+        self.assertRaises(PackageRepo.UnknownCurrentBranchException, self.repo.getCurrentBranch)
+
+class TestNano(PackageRepoTestCase):
+    package = Package('nano')
+    package.L1 = '3dcfa09f5217eedf6ec7539af7e243655d3abdb6' # 3.2-54
+    package.L2 = 'b8243dd54e8feb16a11474f848b8735f5591cf12' # 3.2-55
+
+    def testGetNVR(self):
+        nvr = self.repo.getNVR(self.L2)
+        self.assertEqual(nvr, ('nano', '3.2', '55'))
+
+class TestMySQL_Python(PackageRepoTestCase):
+    package = Package('MySQL-python')
+    package.L1 = '386163d8fc9c857c7194c4e958374af4c4f071ed' # 1.2.5-31
+    package.L2 = 'f85bc5ec2141384f45f224d4464a0a44a981a4d4' # 1.2.5-33
+
+    def testGetNVR(self):
+        nvr = self.repo.getNVR(self.L2)
+        self.assertEqual(nvr, ('MySQL-python', '1.2.5', '33'))

patchfilter.py

@@ -0,0 +1,361 @@
+#!/usr/bin/python3
+import sys
+import tempfile
+import subprocess
+
+header = list()
+files = list()
+files_chunks = dict()	# dict to list
+files_header = dict()	# dict to list
+
+
+
+def push_chunk(file, chunk):
+    global files_chunks
+    
+    if len(chunk) == 0:
+        return
+    
+    if file not in files_chunks:
+        files_chunks[file] = list()
+        
+    files_chunks[file].append(chunk)
+
+
+def parse_patch(lines):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    
+    phase = 0
+    
+    currentfile = ""
+    currentchunk = list()
+    
+        
+    for line in lines:
+        line = line.replace("\n","")
+        
+        if line.startswith("diff --git"):
+        
+            push_chunk(currentfile, currentchunk)
+            currentchunk = list()
+        
+            filename = line.replace("diff --git","")
+            index= filename.find("b/")
+            if index >= 0:
+                filename = filename[index:].strip()
+                currentfile = filename
+                files.append(currentfile)
+                phase = 1
+                
+        if line.startswith("@@"):
+            push_chunk(currentfile, currentchunk)
+            currentchunk = list()
+            phase = 2
+            
+            
+        if phase == 0:
+            header.append(line)
+        if phase == 1:
+            if currentfile not in files_header:
+                files_header[currentfile] = list()
+            files_header[currentfile].append(line) 
+        if phase == 2:
+            currentchunk.append(line)
+        
+    push_chunk(currentfile, currentchunk)
+    currentchunk = list()
+
+def print_all():
+    global header
+    global files
+    global files_chunks
+    global files_header
+
+    for line in header:
+        print(line)
+    for file in files:
+        for line in files_header[file]:
+            print(line)
+        if file in files_chunks:
+            for chunk in files_chunks[file]:
+                for line in chunk:
+                    print(line)
+                    
+                    
+def zap_entire_file(filename):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    if filename in files:
+        files.remove(filename)                
+
+def zap_entire_file_end(filename):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            files.remove(file)                
+            
+def zap_line_in_file(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[filename]:
+                to_delete = list()
+                for line in chunk:
+                    if line == ("+" + match):
+                        to_delete.append(line)
+                    if line == ("-" + match):
+                        to_delete.append(line)
+                for line in to_delete:
+                    chunk.remove(line)
+            
+
+def zap_line_in_file_start(filename, match):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_remove = list()
+                for line in chunk:
+                    if line.startswith("+" + match):
+                        to_remove.append(line)
+                    if line.startswith("-" + match):
+                        to_remove.append(line)
+                for line in to_remove:
+                    chunk.remove(line)
+            
+def zap_plus_line_in_file(filename):
+    global header
+    global files
+    global files_chunks
+    global files_header
+    
+    for file in files:
+        if file.endswith(filename):
+            for chunk in files_chunks[file]:
+                to_remove = list()
+                for line in chunk:
+                    if line.startswith("+"):
+                        to_remove.append(line)
+                for line in to_remove:
+                    chunk.remove(line)
+            
+
+
+def chunk_empty(chunk):
+    for line in chunk:
+        if len(line) > 0 and line[0] == '+':
+            return False
+    return True
+                           
+def zap_empty_chunks():
+    global header
+    global files
+    global files_chunks
+    global files_header
+    files_to_remove = list()
+    for file in files:
+        if file not in files_chunks:
+            files.remove(file)
+            continue
+        to_remove = list()
+        for chunk in files_chunks[file]:
+            if chunk_empty(chunk):
+                to_remove.append(chunk)
+        for c in to_remove:
+            files_chunks[file].remove(c)
+        if len(files_chunks[file]) == 0:
+            files_to_remove.append(file)
+            
+    for file in files_to_remove:
+        files.remove(file)
+            
+            
+
+
+def main():
+
+    if len(sys.argv) > 1:
+        filename = sys.argv[1]
+    
+        with open (filename, "r") as myfile:
+            lines = myfile.readlines()
+    else:
+        output = subprocess.check_output("git format-patch -1 --stdout", shell=True).decode("latin-1")
+        lines = output.split("\n")
+        
+    parse_patch(lines)
+
+    zap_entire_file("b/release")
+    zap_entire_file("b/buildreq_cache")
+    zap_entire_file("b/.gitignore")
+    zap_entire_file("b/upstream")
+    zap_entire_file("b/NEWS")
+    zap_entire_file("b/ChangeLog")
+    zap_entire_file_end("xz.sig")
+    zap_entire_file_end("gz.sig")
+    zap_entire_file_end("bz2.sig")
+    
+    zap_line_in_file("b/testresults", "Total : 0")
+    zap_line_in_file("b/testresults", "Pass : 0")
+    zap_line_in_file("b/testresults", "Fail : 0")
+    zap_line_in_file("b/testresults", "XFail : 0")
+    zap_line_in_file("b/testresults", "Skip : 0")
+    
+    zap_line_in_file("b/requires_ban", "#FOO")
+    zap_line_in_file("b/Makefile", "ARCHIVES = ")
+    zap_line_in_file("b/Makefile", "include ../common/Makefile.common")
+    zap_line_in_file("b/Makefile", "")
+    
+    zap_line_in_file_start(".spec", "Release  :")
+    zap_line_in_file_start(".spec", "Source99 :")
+    zap_line_in_file_start(".spec", "Source0  :")
+    zap_line_in_file_start(".spec", "export SOURCE_DATE_EPOCH")
+    zap_line_in_file_start(".spec", "export GCC_IGNORE_WERROR=1")
+    zap_line_in_file_start(".spec", "export AR=gcc-ar")
+    zap_line_in_file_start(".spec", "export RANLIB=gcc-ranlib")
+    zap_line_in_file_start(".spec", "export NM=gcc-nm")
+    zap_line_in_file_start(".spec", "export CFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FCFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export FFLAGS=\"$CFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "export CXXFLAGS=\"$CXXFLAGS -O3 -ffat-lto-objects -flto=4 \"")
+    zap_line_in_file_start(".spec", "%setup -q -n ")
+    zap_line_in_file_start(".spec", "URL      :")
+    zap_line_in_file_start(".spec", "Version  :")
+    zap_line_in_file_start(".spec", "Group    : Development/Tools")
+    zap_line_in_file_start(".spec", "No detailed description available")
+    zap_line_in_file_start(".spec", "bin components for the")
+    zap_line_in_file_start(".spec", "dev components for the")
+    zap_line_in_file_start(".spec", "lib components for the")
+    zap_line_in_file_start(".spec", "data components for the")
+    zap_line_in_file_start(".spec", "locales components for the")
+    zap_line_in_file_start(".spec", "license components for the")
+    zap_line_in_file_start(".spec", "doc components for the")
+    zap_line_in_file_start(".spec", "Group: Default")
+    zap_line_in_file_start(".spec", "export http_proxy=http://127.0.0.1:9/")
+    zap_line_in_file_start(".spec", "export https_proxy=http://127.0.0.1:9/")
+    zap_line_in_file_start(".spec", "export ftp_proxy=http://127.0.0.1:9/")
+    zap_line_in_file_start(".spec", "export no_proxy=localhost,127.0.0.1,0.0.0.0")
+    zap_line_in_file_start(".spec", "Summary:")
+    zap_line_in_file_start(".spec", "Group: Development")
+    zap_line_in_file_start(".spec", "Group: Binaries")
+    zap_line_in_file_start(".spec", "Group: Data")
+    zap_line_in_file_start(".spec", "Group: Libraries")
+    zap_line_in_file_start(".spec", "Group: Documentation")
+    zap_line_in_file_start(".spec", "export LANG=C")
+    zap_line_in_file_start(".spec", "%description lib")
+    zap_line_in_file_start(".spec", "%description bin")
+    zap_line_in_file_start(".spec", "%description data")
+    zap_line_in_file_start(".spec", "%description doc")
+    zap_line_in_file_start(".spec", "%description locales")
+    zap_line_in_file_start(".spec", "%description dev")
+    zap_line_in_file_start(".spec", "%description license")
+    zap_line_in_file_start(".spec", "%package doc")
+    zap_line_in_file_start(".spec", "cp -a ")
+ 
+    zap_line_in_file_start("b/Makefile", "URL =")
+    zap_line_in_file_start("b/Makefile", "PKG_NAME :=")
+
+    zap_line_in_file("b/options.conf", "archives = ")
+    zap_line_in_file("b/options.conf", "giturl = ")
+    zap_line_in_file("b/options.conf", "[package]")
+    zap_line_in_file("b/options.conf", "")
+    zap_line_in_file("b/options.conf", "[autospec]")
+    zap_line_in_file("b/options.conf", "# build 32 bit libraries")
+    zap_line_in_file("b/options.conf", "32bit = false")
+    zap_line_in_file("b/options.conf", "# allow package to build with test failures")
+    zap_line_in_file("b/options.conf", "allow_test_failures = false")
+    zap_line_in_file("b/options.conf", "# unset %build ld_as_needed variable")
+    zap_line_in_file("b/options.conf", "asneeded = false")
+    zap_line_in_file("b/options.conf", "# this package is trusted enough to automatically update (used by other tools)")
+    zap_line_in_file("b/options.conf", "autoupdate = false")
+    zap_line_in_file("b/options.conf", "# extend flags with '-std=gnu++98")
+    zap_line_in_file("b/options.conf", "broken_c++ = false")
+    zap_line_in_file("b/options.conf", "# disable parallelization during build")
+    zap_line_in_file("b/options.conf", "broken_parallel_build = false")
+    zap_line_in_file("b/options.conf", "# this package is a library compatability package and only ships versioned library files")
+    zap_line_in_file("b/options.conf", "compat = false")
+    zap_line_in_file("b/options.conf", "# set conservative build flags")
+    zap_line_in_file("b/options.conf", "conservative_flags = false")
+    zap_line_in_file("b/options.conf", "# dev package requires the extras to be installed")
+    zap_line_in_file("b/options.conf", "dev_requires_extras = false")
+    zap_line_in_file("b/options.conf", "# pass -ffast-math to compiler")
+    zap_line_in_file("b/options.conf", "fast-math = false")
+    zap_line_in_file("b/options.conf", "# optimize build for speed over size")
+    zap_line_in_file("b/options.conf", "funroll-loops = false")
+    zap_line_in_file("b/options.conf", "# set flags to smallest -02 flags possible")
+    zap_line_in_file("b/options.conf", "insecure_build = false")
+    zap_line_in_file("b/options.conf", "# do not remove static libraries")
+    zap_line_in_file("b/options.conf", "keepstatic = false")
+    zap_line_in_file("b/options.conf", "# do not require autostart subpackage")
+    zap_line_in_file("b/options.conf", "no_autostart = false")
+    zap_line_in_file("b/options.conf", "# disable stripping binaries")
+    zap_line_in_file("b/options.conf", "nostrip = false")
+    zap_line_in_file("b/options.conf", "# optimize build for size over speed")
+    zap_line_in_file("b/options.conf", "optimize_size = false")
+    zap_line_in_file("b/options.conf", "# set profile for pgo")
+    zap_line_in_file("b/options.conf", "pgo = false")
+    zap_line_in_file("b/options.conf", "# set flags for security-sensitive builds")
+    zap_line_in_file("b/options.conf", "security_sensitive = false")
+    zap_line_in_file("b/options.conf", "# do not run test suite")
+    zap_line_in_file("b/options.conf", "skip_tests = false")
+    zap_line_in_file("b/options.conf", "# add .so files to the lib package instead of dev")
+    zap_line_in_file("b/options.conf", "so_to_lib = false")
+    zap_line_in_file("b/options.conf", "# configure build for avx2")
+    zap_line_in_file("b/options.conf", "use_avx2 = false")
+    zap_line_in_file("b/options.conf", "# configure build for avx512")
+    zap_line_in_file("b/options.conf", "use_avx512 = false")
+    zap_line_in_file("b/options.conf", "# add clang flags")
+    zap_line_in_file("b/options.conf", "use_clang = false")
+    zap_line_in_file("b/options.conf", "# configure build for lto")
+    zap_line_in_file("b/options.conf", "use_lto = false")
+    zap_line_in_file("b/options.conf", "# require package verification for build")
+    zap_line_in_file("b/options.conf", "verify_required = true")
+
+    zap_line_in_file("b/buildreq_add", "# This file contains additional build requirements that did not get")
+    zap_line_in_file("b/buildreq_add", "# picked up automatically. One name per line, no whitespace.")
+    zap_line_in_file("b/buildreq_ban", "# This file contains build requirements that get picked up but are")
+    zap_line_in_file("b/buildreq_ban", "# undesirable. One entry per line, no whitespace.")
+    
+    
+    zap_line_in_file("b/excludes", "# This file contains the output files that need %exclude. Full path")
+    zap_line_in_file("b/excludes", "# names, one per line.")
+
+    zap_line_in_file("b/pkgconfig_add", "# This file contains additional pkgconfig build requirements that did")
+    zap_line_in_file("b/pkgconfig_add", "# not get picked up automatically. One name per line, no whitespace.")
+    zap_line_in_file("b/pkgconfig_ban", "# This file contains pkgconfig build requirements that get picked up")
+    zap_line_in_file("b/pkgconfig_ban", "# but are undesirable. One entry per line, no whitespace.")
+    zap_line_in_file("b/requires_add", "# This file contains additional runtime requirements that did not get")
+    zap_line_in_file("b/requires_add", "# picked up automatically. One name per line, no whitespace.")
+    zap_line_in_file("b/requires_ban", "# This file contains runtime requirements that get picked up but are")
+    zap_line_in_file("b/requires_ban", "# undesirable. One entry per line, no whitespace.")
+    
+    zap_line_in_file_start(".spec", "Summary  : No detailed summary available")
+    
+    zap_plus_line_in_file("symbols")
+    zap_plus_line_in_file("symbols32")
+
+    zap_entire_file("b/whatrequires")
+
+    zap_empty_chunks()
+    
+    print_all()
+
+if __name__ == '__main__':
+    with tempfile.TemporaryDirectory() as workingdir:
+        main()
+

user-setup.sh

@@ -145,9 +145,9 @@ if [ "$USE_KOJI" ]; then
   cp "$CLIENTCA" ~/.koji/clientca.crt
   cp "$SERVERCA" ~/.koji/serverca.crt
 
-  if [ ! -f /etc/koji.conf ]; then
+  if [ ! -f ~/.koji/config ]; then
     echo "Setting up koji config . . ."
-    sudo cp projects/common/conf/koji.conf /etc
+    cp -f projects/common/conf/koji.conf ~/.koji/config
   fi
 fi