LTS package maintenance utility (ltsutils): initial commit

This tooling is designed to automate 2 main tasks that are part of
the package maintenance workflow of Clear Linux LTS. These tasks are:
- Back-porting of a patch (e.g. security fix) to older branches.
- (Not implemented yet) Building RPMs with the intent of sharing binaries of
  older LTS branches to newer branches whenever possible.

2 new targets are defined in Makefile.common.lts:
- lts-show: Show a summary of active LTS branches
- lts-backport: Attempt to fast-forward the previous active branch to the current branch

"Active" branches correspond to LTS releases that currently have support.
They are listed in a flat file "active-branches" in "lts" directory, from
oldest to newest. New entries are added by Clear Linux LTS developers as
new releases become available, and entries removed as releases become
obsolete.

Note: For CVE patching, the tool is not aware of CVE severity levels or
the minimum supported severity level of each LTS branch. For now it is
the user's responsibility to know when a CVE does not apply to older
branches and stop calling "make lts-backport".

Signed-off-by: Tan, Yew Wayne <yew.wayne.tan@intel.com>

Makefile.common

@@ -503,6 +503,9 @@ cloc: $(SRPMFILE)
 	@$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME)
 	cat results/cloc.txt
 
+# Define LTS-specific targets in a separate makefile
+-include $(TOPLVL)/projects/common/Makefile.common.lts
+
 # Define site local common targets in a separate makefile
 -include $(TOPLVL)/projects/common/Makefile.common.site_local
 

Makefile.common.lts

@@ -0,0 +1,31 @@
+#-*-makefile-*-
+
+LTSUTILS = python $(TOPLVL)/projects/common/lts/main.py $(PKG_NAME)
+
+#help lts-show: Display a summary of active LTS branches.
+lts-show:
+	@$(LTSUTILS) sanity-check
+	@while read b; do \
+		if git show-ref $$b > /dev/null; then \
+			echo $$b $$(git log --oneline -1 $$b); \
+		else \
+			echo $$b Not found; \
+		fi; \
+	done < $(TOPLVL)/projects/common/lts/active-branches
+
+#help lts-backport: Fast-forward the previous active branch to the current
+#help branch.
+lts-backport:
+	@$(LTSUTILS) sanity-check
+	@newer=$$(git symbolic-ref HEAD); \
+	newer=$${newer#refs/heads/}; \
+	if ! $(LTSUTILS) checkout-prev; then \
+		echo Could not check out previous active branch.; \
+		exit 0; \
+	fi; \
+	if $(LTSUTILS) is-same-version $$newer; then \
+		$(LTSUTILS) fast-forward $$newer; \
+	else \
+		echo Most likely a patch needs to be manually re-applied for this version.; \
+		echo Alternatively, use \"git merge --ff-only $$newer\" to upgrade the package version.; \
+	fi

lts/Makefile

@@ -0,0 +1,5 @@
+TEST = test.test_ltsutils
+
+.PHONY: test
+test:
+	PYTHONPATH=. python -m unittest -v $(TEST)

lts/README.md

@@ -0,0 +1,18 @@
+# LTS package maintenance utility
+
+This tooling is designed to automate 2 main tasks that are part of
+the package maintenance workflow of Clear Linux LTS. These tasks are:
+- Back-porting of a patch (e.g. security fix) to older branches.
+- Building RPMs with the intent of sharing binaries of older LTS branches to
+  newer branches whenever possible.
+
+There should be no need to run this tool directly. Instead use the following
+targets defined in Makefile.common.lts:
+- lts-show: Show a summary of active LTS branches
+- lts-backport: Attempt to fast-forward the previous active branch to the current branch
+
+"Active" branches correspond to LTS releases that currently have support.
+They are listed in a flat file "active-branches" in "lts" directory, from
+oldest to newest. New entries are added by Clear Linux LTS developers as
+new releases become available, and entries removed as releases become
+obsolete.

lts/ltsutils/package_repo.py

@@ -0,0 +1,52 @@
+import pathlib
+from subprocess import PIPE, CalledProcessError
+from .shell import Shell
+
+class PackageRepo:
+    '''Represents a package repository. Most methods are wrappers of git commands.'''
+    class UnknownCurrentBranchException(Exception): pass
+    class InvalidBranchException(Exception): pass
+
+    def __init__(self, name, path):
+        self.name = name
+        self.path = pathlib.Path(path)
+        self.sh = Shell(self.path)
+
+    def getNVR(self, commit='HEAD'):
+        with self.sh.popen(['git', 'show', '{}:{}.spec'.format(commit, self.name)], stdout=PIPE) as specfile:
+            nvr = self.sh.run('rpmspec --srpm -q --queryformat %{NVR} /dev/stdin', stdin=specfile.stdout)
+        return tuple(nvr.stdout.strip().split('-'))
+
+    def checkoutBranch(self, branch, allow_remote=False):
+        # allow_remote=True allows checking out a new remote-tracking branch
+        if not allow_remote and not self.hasBranch(branch):
+            raise self.InvalidBranchException(branch)
+        self.sh.run_args(['git', 'checkout', branch], capture_output=False)
+
+    def fastForwardBranch(self, old, new):
+        self.checkoutBranch(old)
+        if not self.hasBranch(new):
+            raise self.InvalidBranchException(new)
+        self.sh.run_args(['git', 'merge', '--ff-only', new], capture_output=False)
+
+    def getActiveBranches(self):
+        toplvl = pathlib.Path(self.path) / '../..'
+        common = toplvl / 'projects/common'
+        active_branches = common / 'lts/active-branches'
+        with active_branches.open() as f:
+            return [line.rstrip() for line in f]
+
+    def getCurrentBranch(self):
+        try:
+            head = self.sh.run('git symbolic-ref HEAD').stdout.strip()
+        except CalledProcessError:
+            raise self.UnknownCurrentBranchException
+
+        refs_heads = 'refs/heads/'
+        assert head.startswith(refs_heads)
+        head = head[len(refs_heads):]
+        return head
+
+    def hasBranch(self, branch):
+        p = self.sh.run_args(['git', 'rev-parse', 'refs/heads/'+branch], check=False)
+        return p.returncode == 0

lts/ltsutils/shell.py

@@ -0,0 +1,27 @@
+import subprocess, shlex
+
+class Shell:
+    # Default options passed to subprocess.run. May be customized per-instance.
+    cwd = None
+    check = True
+    capture_output = True
+    text = True
+
+    def __init__(self, cwd=None):
+        if cwd: self.cwd = cwd
+
+    def run_args(self, args, **kwargs):
+        kwargs1 = {
+                'check': self.check,
+                'capture_output': self.capture_output,
+                'text': self.text,
+                'cwd': self.cwd
+                }
+        kwargs1.update(kwargs)
+        return subprocess.run(args, **kwargs1)
+
+    def run(self, cmd, **kwargs):
+        return self.run_args(shlex.split(cmd), **kwargs)
+
+    def popen(self, args, **kwargs):
+        return subprocess.Popen(args, cwd=self.cwd, **kwargs)

lts/main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+import sys,argparse
+
+from ltsutils.package_repo import PackageRepo
+
+def log(msg, **kwargs):
+    print(msg, file=sys.stderr)
+
+def init_parser():
+    main = argparse.ArgumentParser()
+    main.add_argument('package_name', nargs=1)
+    subparsers = main.add_subparsers(dest='command', metavar='command', required=True)
+
+    p = subparsers.add_parser('checkout-prev',
+            help='checkout previous branch')
+    p = subparsers.add_parser('is-same-version',
+            help='return true if package version is the same as the given branch')
+    p.add_argument('branch', nargs=1)
+    p = subparsers.add_parser('fast-forward',
+            help='fast-forward current branch to a newer branch')
+    p.add_argument('branch', nargs=1)
+
+    p = subparsers.add_parser('sanity-check',
+            help='run sanity checks for data consistency')
+
+    return main
+
+def checkout_prev(args, repo):
+    active_branches = repo.getActiveBranches()
+    current = repo.getCurrentBranch()
+
+    i = active_branches.index(current)
+    if i == 0:
+        log('Already on oldest active branch.')
+        return False
+
+    prev = active_branches[i-1]
+    repo.checkoutBranch(prev, allow_remote=True)
+
+def is_same_version(args, other):
+    current = repo.getCurrentBranch()
+    other = args.branch[0]
+    assert repo.hasBranch(other), 'Branch %s not found' % other
+
+    v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (current, other)]
+    if v1 != v2:
+        log('Current version {} does not match version {} on branch {}'.format(v1, v2, other))
+    return v1 == v2
+
+def fast_forward(args, repo):
+    current = repo.getCurrentBranch()
+    newer = args.branch[0]
+
+    log('Fast-forwarding {} to {}'.format(current, newer))
+    repo.fastForwardBranch(current, newer)
+
+def sanity_check(args, repo):
+    ok = True
+    # HEAD must point to a branch
+    try:
+        current = repo.getCurrentBranch()
+    except PackageRepo.UnknownCurrentBranchException:
+        log('Unknown current branch. Has a branch been checked out?')
+        current = None
+        ok = False
+
+    # active-branches file must not be empty
+    active_branches = repo.getActiveBranches()
+    if not len(active_branches):
+        log('No active branches defined. Is active-branches file empty?')
+        ok = False
+    # current branch must be an active branch
+    elif current and current not in active_branches:
+        log('%s is not an active LTS branch.' % current)
+        ok = False
+
+    return ok
+
+if __name__=='__main__':
+    args = init_parser().parse_args()
+    repo = PackageRepo(args.package_name[0], '.')
+
+    commands = {
+            'checkout-prev': checkout_prev,
+            'is-same-version': is_same_version,
+            'fast-forward': fast_forward,
+            'sanity-check': sanity_check,
+            }
+    ret = commands[args.command](args, repo)
+    if ret is not None:
+        exit(0 if ret else 1)

lts/test/test_ltsutils.py

@@ -0,0 +1,79 @@
+import unittest
+import os, pathlib, tempfile, logging
+import subprocess, functools
+
+from ltsutils.package_repo import PackageRepo
+import ltsutils.shell
+
+run = functools.partial(subprocess.run, check=True)
+
+class LTSUtilsTestCase(unittest.TestCase):
+    toplvl = pathlib.Path('../../..')
+    packages = toplvl / 'packages'
+    pkgname = 'nano'
+    master_commit = '6947e7170435e179aaa86b156e53cae644ce1d73'
+    repo_url = 'https://github.com/clearlinux-pkgs/{}.git'.format(pkgname)
+
+    def cloneOrExtractRepo(self):
+        tmpdir = pathlib.Path('/var/tmp/common-lts-test')
+        tmpdir.mkdir(mode=0o700, exist_ok=True)
+        tarball = tmpdir / '{}.tar.gz'.format(self.pkgname)
+        if tarball.exists():
+            run(['tar', 'xf', tarball, '-C', self.workdir])
+        else:
+            run(['git', 'clone', self.repo_url, self.workdir])
+            run(['tar', 'czf', tarball, '-C', self.workdir, '.'])
+
+    def setUp(self):
+        self._tmpdir = tempfile.TemporaryDirectory(prefix='test-{}-'.format(self.pkgname), dir=self.packages)
+        self.workdir = pathlib.Path(self._tmpdir.name)
+        self.cloneOrExtractRepo()
+        self.repo = PackageRepo(self.pkgname, self.workdir)
+
+        self._sh = ltsutils.shell.Shell(self.workdir)
+        self._sh.capture_output = False
+
+    def sh(self, cmd):
+        return self._sh.run(cmd)
+
+    def sh_stdout(self, cmd):
+        return self._sh.run(cmd, capture_output=True).stdout.rstrip()
+
+    def tearDown(self):
+        self._tmpdir.cleanup()
+
+class TestPackageRepo(LTSUtilsTestCase):
+    L1 = '3dcfa09f5217eedf6ec7539af7e243655d3abdb6' # 3.2-54
+    L2 = 'b8243dd54e8feb16a11474f848b8735f5591cf12' # 3.2-55
+
+    def setUp(self):
+        super().setUp()
+        self.sh('git branch L1 %s' % self.L1)
+        self.sh('git branch L2 %s' % self.L2)
+
+    def testGetNVR(self):
+        nvr = self.repo.getNVR(self.L2)
+        self.assertEqual(nvr, ('nano', '3.2', '55'))
+
+    def testHasBranch(self):
+        self.assertTrue(self.repo.hasBranch('L2'))
+        self.assertFalse(self.repo.hasBranch('L3'))
+
+    def testCheckoutBranch(self):
+        self.repo.checkoutBranch('L2')
+        self.assertEqual(self.sh_stdout('git rev-parse HEAD'), self.L2)
+
+        self.assertRaises(PackageRepo.InvalidBranchException, self.repo.checkoutBranch, 'L3')
+
+    def testFastForward(self):
+        self.repo.fastForwardBranch('L1', 'L2')
+        self.assertEqual(self.sh_stdout('git rev-parse L1'), self.L2)
+        self.assertEqual(self.sh_stdout('git rev-parse L2'), self.L2)
+
+    def testGetCurrentBranch(self):
+        self.repo.checkoutBranch('L2')
+        b = self.repo.getCurrentBranch()
+        self.assertEqual(b, 'L2')
+
+        self.sh('git checkout --detach L2')
+        self.assertRaises(PackageRepo.UnknownCurrentBranchException, self.repo.getCurrentBranch)