From 69cee2fcd7f7abb3ead7649e270972041f5c16fa Mon Sep 17 00:00:00 2001 From: "Tan, Yew Wayne" Date: Tue, 30 Apr 2019 15:15:57 +0800 Subject: [PATCH] LTS package maintenance utility (ltsutils): initial commit This tooling is designed to automate 2 main tasks that are part of the package maintenance workflow of Clear Linux LTS. These tasks are: - Back-porting of a patch (e.g. security fix) to older branches. - (Not implemented yet) Building RPMs with the intent of sharing binaries of older LTS branches to newer branches whenever possible. 2 new targets are defined in Makefile.common.lts: - lts-show: Show a summary of active LTS branches - lts-backport: Attempt to fast-forward the previous active branch to the current branch "Active" branches correspond to LTS releases that currently have support. They are listed in a flat file "active-branches" in "lts" directory, from oldest to newest. New entries are added by Clear Linux LTS developers as new releases become available, and entries removed as releases become obsolete. Note: For CVE patching, the tool is not aware of CVE severity levels or the minimum supported severity level of each LTS branch. For now it is the user's responsibility to know when a CVE does not apply to older branches and stop calling "make lts-backport". Signed-off-by: Tan, Yew Wayne --- Makefile.common | 3 ++ Makefile.common.lts | 31 ++++++++++++ lts/Makefile | 5 ++ lts/README.md | 18 +++++++ lts/active-branches | 0 lts/ltsutils/__init__.py | 0 lts/ltsutils/package_repo.py | 52 +++++++++++++++++++++ lts/ltsutils/shell.py | 27 +++++++++++ lts/main.py | 91 ++++++++++++++++++++++++++++++++++++ lts/test/__init__.py | 0 lts/test/test_ltsutils.py | 79 +++++++++++++++++++++++++++++++ 11 files changed, 306 insertions(+) create mode 100644 Makefile.common.lts create mode 100644 lts/Makefile create mode 100644 lts/README.md create mode 100644 lts/active-branches create mode 100644 lts/ltsutils/__init__.py create mode 100644 lts/ltsutils/package_repo.py create mode 100644 lts/ltsutils/shell.py create mode 100644 lts/main.py create mode 100644 lts/test/__init__.py create mode 100644 lts/test/test_ltsutils.py diff --git a/Makefile.common b/Makefile.common index dc7de83..92ad8f5 100644 --- a/Makefile.common +++ b/Makefile.common @@ -503,6 +503,9 @@ cloc: $(SRPMFILE) @$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME) cat results/cloc.txt +# Define LTS-specific targets in a separate makefile +-include $(TOPLVL)/projects/common/Makefile.common.lts + # Define site local common targets in a separate makefile -include $(TOPLVL)/projects/common/Makefile.common.site_local diff --git a/Makefile.common.lts b/Makefile.common.lts new file mode 100644 index 0000000..581297f --- /dev/null +++ b/Makefile.common.lts @@ -0,0 +1,31 @@ +#-*-makefile-*- + +LTSUTILS = python $(TOPLVL)/projects/common/lts/main.py $(PKG_NAME) + +#help lts-show: Display a summary of active LTS branches. +lts-show: + @$(LTSUTILS) sanity-check + @while read b; do \ + if git show-ref $$b > /dev/null; then \ + echo $$b $$(git log --oneline -1 $$b); \ + else \ + echo $$b Not found; \ + fi; \ + done < $(TOPLVL)/projects/common/lts/active-branches + +#help lts-backport: Fast-forward the previous active branch to the current +#help branch. +lts-backport: + @$(LTSUTILS) sanity-check + @newer=$$(git symbolic-ref HEAD); \ + newer=$${newer#refs/heads/}; \ + if ! $(LTSUTILS) checkout-prev; then \ + echo Could not check out previous active branch.; \ + exit 0; \ + fi; \ + if $(LTSUTILS) is-same-version $$newer; then \ + $(LTSUTILS) fast-forward $$newer; \ + else \ + echo Most likely a patch needs to be manually re-applied for this version.; \ + echo Alternatively, use \"git merge --ff-only $$newer\" to upgrade the package version.; \ + fi diff --git a/lts/Makefile b/lts/Makefile new file mode 100644 index 0000000..aaee1f8 --- /dev/null +++ b/lts/Makefile @@ -0,0 +1,5 @@ +TEST = test.test_ltsutils + +.PHONY: test +test: + PYTHONPATH=. python -m unittest -v $(TEST) diff --git a/lts/README.md b/lts/README.md new file mode 100644 index 0000000..51358e3 --- /dev/null +++ b/lts/README.md @@ -0,0 +1,18 @@ +# LTS package maintenance utility + +This tooling is designed to automate 2 main tasks that are part of +the package maintenance workflow of Clear Linux LTS. These tasks are: +- Back-porting of a patch (e.g. security fix) to older branches. +- Building RPMs with the intent of sharing binaries of older LTS branches to + newer branches whenever possible. + +There should be no need to run this tool directly. Instead use the following +targets defined in Makefile.common.lts: +- lts-show: Show a summary of active LTS branches +- lts-backport: Attempt to fast-forward the previous active branch to the current branch + +"Active" branches correspond to LTS releases that currently have support. +They are listed in a flat file "active-branches" in "lts" directory, from +oldest to newest. New entries are added by Clear Linux LTS developers as +new releases become available, and entries removed as releases become +obsolete. diff --git a/lts/active-branches b/lts/active-branches new file mode 100644 index 0000000..e69de29 diff --git a/lts/ltsutils/__init__.py b/lts/ltsutils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lts/ltsutils/package_repo.py b/lts/ltsutils/package_repo.py new file mode 100644 index 0000000..7dcae87 --- /dev/null +++ b/lts/ltsutils/package_repo.py @@ -0,0 +1,52 @@ +import pathlib +from subprocess import PIPE, CalledProcessError +from .shell import Shell + +class PackageRepo: + '''Represents a package repository. Most methods are wrappers of git commands.''' + class UnknownCurrentBranchException(Exception): pass + class InvalidBranchException(Exception): pass + + def __init__(self, name, path): + self.name = name + self.path = pathlib.Path(path) + self.sh = Shell(self.path) + + def getNVR(self, commit='HEAD'): + with self.sh.popen(['git', 'show', '{}:{}.spec'.format(commit, self.name)], stdout=PIPE) as specfile: + nvr = self.sh.run('rpmspec --srpm -q --queryformat %{NVR} /dev/stdin', stdin=specfile.stdout) + return tuple(nvr.stdout.strip().split('-')) + + def checkoutBranch(self, branch, allow_remote=False): + # allow_remote=True allows checking out a new remote-tracking branch + if not allow_remote and not self.hasBranch(branch): + raise self.InvalidBranchException(branch) + self.sh.run_args(['git', 'checkout', branch], capture_output=False) + + def fastForwardBranch(self, old, new): + self.checkoutBranch(old) + if not self.hasBranch(new): + raise self.InvalidBranchException(new) + self.sh.run_args(['git', 'merge', '--ff-only', new], capture_output=False) + + def getActiveBranches(self): + toplvl = pathlib.Path(self.path) / '../..' + common = toplvl / 'projects/common' + active_branches = common / 'lts/active-branches' + with active_branches.open() as f: + return [line.rstrip() for line in f] + + def getCurrentBranch(self): + try: + head = self.sh.run('git symbolic-ref HEAD').stdout.strip() + except CalledProcessError: + raise self.UnknownCurrentBranchException + + refs_heads = 'refs/heads/' + assert head.startswith(refs_heads) + head = head[len(refs_heads):] + return head + + def hasBranch(self, branch): + p = self.sh.run_args(['git', 'rev-parse', 'refs/heads/'+branch], check=False) + return p.returncode == 0 diff --git a/lts/ltsutils/shell.py b/lts/ltsutils/shell.py new file mode 100644 index 0000000..f3aa16c --- /dev/null +++ b/lts/ltsutils/shell.py @@ -0,0 +1,27 @@ +import subprocess, shlex + +class Shell: + # Default options passed to subprocess.run. May be customized per-instance. + cwd = None + check = True + capture_output = True + text = True + + def __init__(self, cwd=None): + if cwd: self.cwd = cwd + + def run_args(self, args, **kwargs): + kwargs1 = { + 'check': self.check, + 'capture_output': self.capture_output, + 'text': self.text, + 'cwd': self.cwd + } + kwargs1.update(kwargs) + return subprocess.run(args, **kwargs1) + + def run(self, cmd, **kwargs): + return self.run_args(shlex.split(cmd), **kwargs) + + def popen(self, args, **kwargs): + return subprocess.Popen(args, cwd=self.cwd, **kwargs) diff --git a/lts/main.py b/lts/main.py new file mode 100644 index 0000000..a579f7d --- /dev/null +++ b/lts/main.py @@ -0,0 +1,91 @@ +#!/usr/bin/python +import sys,argparse + +from ltsutils.package_repo import PackageRepo + +def log(msg, **kwargs): + print(msg, file=sys.stderr) + +def init_parser(): + main = argparse.ArgumentParser() + main.add_argument('package_name', nargs=1) + subparsers = main.add_subparsers(dest='command', metavar='command', required=True) + + p = subparsers.add_parser('checkout-prev', + help='checkout previous branch') + p = subparsers.add_parser('is-same-version', + help='return true if package version is the same as the given branch') + p.add_argument('branch', nargs=1) + p = subparsers.add_parser('fast-forward', + help='fast-forward current branch to a newer branch') + p.add_argument('branch', nargs=1) + + p = subparsers.add_parser('sanity-check', + help='run sanity checks for data consistency') + + return main + +def checkout_prev(args, repo): + active_branches = repo.getActiveBranches() + current = repo.getCurrentBranch() + + i = active_branches.index(current) + if i == 0: + log('Already on oldest active branch.') + return False + + prev = active_branches[i-1] + repo.checkoutBranch(prev, allow_remote=True) + +def is_same_version(args, other): + current = repo.getCurrentBranch() + other = args.branch[0] + assert repo.hasBranch(other), 'Branch %s not found' % other + + v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (current, other)] + if v1 != v2: + log('Current version {} does not match version {} on branch {}'.format(v1, v2, other)) + return v1 == v2 + +def fast_forward(args, repo): + current = repo.getCurrentBranch() + newer = args.branch[0] + + log('Fast-forwarding {} to {}'.format(current, newer)) + repo.fastForwardBranch(current, newer) + +def sanity_check(args, repo): + ok = True + # HEAD must point to a branch + try: + current = repo.getCurrentBranch() + except PackageRepo.UnknownCurrentBranchException: + log('Unknown current branch. Has a branch been checked out?') + current = None + ok = False + + # active-branches file must not be empty + active_branches = repo.getActiveBranches() + if not len(active_branches): + log('No active branches defined. Is active-branches file empty?') + ok = False + # current branch must be an active branch + elif current and current not in active_branches: + log('%s is not an active LTS branch.' % current) + ok = False + + return ok + +if __name__=='__main__': + args = init_parser().parse_args() + repo = PackageRepo(args.package_name[0], '.') + + commands = { + 'checkout-prev': checkout_prev, + 'is-same-version': is_same_version, + 'fast-forward': fast_forward, + 'sanity-check': sanity_check, + } + ret = commands[args.command](args, repo) + if ret is not None: + exit(0 if ret else 1) diff --git a/lts/test/__init__.py b/lts/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lts/test/test_ltsutils.py b/lts/test/test_ltsutils.py new file mode 100644 index 0000000..1a11d31 --- /dev/null +++ b/lts/test/test_ltsutils.py @@ -0,0 +1,79 @@ +import unittest +import os, pathlib, tempfile, logging +import subprocess, functools + +from ltsutils.package_repo import PackageRepo +import ltsutils.shell + +run = functools.partial(subprocess.run, check=True) + +class LTSUtilsTestCase(unittest.TestCase): + toplvl = pathlib.Path('../../..') + packages = toplvl / 'packages' + pkgname = 'nano' + master_commit = '6947e7170435e179aaa86b156e53cae644ce1d73' + repo_url = 'https://github.com/clearlinux-pkgs/{}.git'.format(pkgname) + + def cloneOrExtractRepo(self): + tmpdir = pathlib.Path('/var/tmp/common-lts-test') + tmpdir.mkdir(mode=0o700, exist_ok=True) + tarball = tmpdir / '{}.tar.gz'.format(self.pkgname) + if tarball.exists(): + run(['tar', 'xf', tarball, '-C', self.workdir]) + else: + run(['git', 'clone', self.repo_url, self.workdir]) + run(['tar', 'czf', tarball, '-C', self.workdir, '.']) + + def setUp(self): + self._tmpdir = tempfile.TemporaryDirectory(prefix='test-{}-'.format(self.pkgname), dir=self.packages) + self.workdir = pathlib.Path(self._tmpdir.name) + self.cloneOrExtractRepo() + self.repo = PackageRepo(self.pkgname, self.workdir) + + self._sh = ltsutils.shell.Shell(self.workdir) + self._sh.capture_output = False + + def sh(self, cmd): + return self._sh.run(cmd) + + def sh_stdout(self, cmd): + return self._sh.run(cmd, capture_output=True).stdout.rstrip() + + def tearDown(self): + self._tmpdir.cleanup() + +class TestPackageRepo(LTSUtilsTestCase): + L1 = '3dcfa09f5217eedf6ec7539af7e243655d3abdb6' # 3.2-54 + L2 = 'b8243dd54e8feb16a11474f848b8735f5591cf12' # 3.2-55 + + def setUp(self): + super().setUp() + self.sh('git branch L1 %s' % self.L1) + self.sh('git branch L2 %s' % self.L2) + + def testGetNVR(self): + nvr = self.repo.getNVR(self.L2) + self.assertEqual(nvr, ('nano', '3.2', '55')) + + def testHasBranch(self): + self.assertTrue(self.repo.hasBranch('L2')) + self.assertFalse(self.repo.hasBranch('L3')) + + def testCheckoutBranch(self): + self.repo.checkoutBranch('L2') + self.assertEqual(self.sh_stdout('git rev-parse HEAD'), self.L2) + + self.assertRaises(PackageRepo.InvalidBranchException, self.repo.checkoutBranch, 'L3') + + def testFastForward(self): + self.repo.fastForwardBranch('L1', 'L2') + self.assertEqual(self.sh_stdout('git rev-parse L1'), self.L2) + self.assertEqual(self.sh_stdout('git rev-parse L2'), self.L2) + + def testGetCurrentBranch(self): + self.repo.checkoutBranch('L2') + b = self.repo.getCurrentBranch() + self.assertEqual(b, 'L2') + + self.sh('git checkout --detach L2') + self.assertRaises(PackageRepo.UnknownCurrentBranchException, self.repo.getCurrentBranch)