diff --git a/Makefile.common b/Makefile.common index dc7de83..92ad8f5 100644 --- a/Makefile.common +++ b/Makefile.common @@ -503,6 +503,9 @@ cloc: $(SRPMFILE) @$(MOCK) --clean --scrub=chroot --uniqueext=$(PKG_NAME) cat results/cloc.txt +# Define LTS-specific targets in a separate makefile +-include $(TOPLVL)/projects/common/Makefile.common.lts + # Define site local common targets in a separate makefile -include $(TOPLVL)/projects/common/Makefile.common.site_local diff --git a/Makefile.common.lts b/Makefile.common.lts new file mode 100644 index 0000000..581297f --- /dev/null +++ b/Makefile.common.lts @@ -0,0 +1,31 @@ +#-*-makefile-*- + +LTSUTILS = python $(TOPLVL)/projects/common/lts/main.py $(PKG_NAME) + +#help lts-show: Display a summary of active LTS branches. +lts-show: + @$(LTSUTILS) sanity-check + @while read b; do \ + if git show-ref $$b > /dev/null; then \ + echo $$b $$(git log --oneline -1 $$b); \ + else \ + echo $$b Not found; \ + fi; \ + done < $(TOPLVL)/projects/common/lts/active-branches + +#help lts-backport: Fast-forward the previous active branch to the current +#help branch. +lts-backport: + @$(LTSUTILS) sanity-check + @newer=$$(git symbolic-ref HEAD); \ + newer=$${newer#refs/heads/}; \ + if ! $(LTSUTILS) checkout-prev; then \ + echo Could not check out previous active branch.; \ + exit 0; \ + fi; \ + if $(LTSUTILS) is-same-version $$newer; then \ + $(LTSUTILS) fast-forward $$newer; \ + else \ + echo Most likely a patch needs to be manually re-applied for this version.; \ + echo Alternatively, use \"git merge --ff-only $$newer\" to upgrade the package version.; \ + fi diff --git a/lts/Makefile b/lts/Makefile new file mode 100644 index 0000000..aaee1f8 --- /dev/null +++ b/lts/Makefile @@ -0,0 +1,5 @@ +TEST = test.test_ltsutils + +.PHONY: test +test: + PYTHONPATH=. python -m unittest -v $(TEST) diff --git a/lts/README.md b/lts/README.md new file mode 100644 index 0000000..51358e3 --- /dev/null +++ b/lts/README.md @@ -0,0 +1,18 @@ +# LTS package maintenance utility + +This tooling is designed to automate 2 main tasks that are part of +the package maintenance workflow of Clear Linux LTS. These tasks are: +- Back-porting of a patch (e.g. security fix) to older branches. +- Building RPMs with the intent of sharing binaries of older LTS branches to + newer branches whenever possible. + +There should be no need to run this tool directly. Instead use the following +targets defined in Makefile.common.lts: +- lts-show: Show a summary of active LTS branches +- lts-backport: Attempt to fast-forward the previous active branch to the current branch + +"Active" branches correspond to LTS releases that currently have support. +They are listed in a flat file "active-branches" in "lts" directory, from +oldest to newest. New entries are added by Clear Linux LTS developers as +new releases become available, and entries removed as releases become +obsolete. diff --git a/lts/active-branches b/lts/active-branches new file mode 100644 index 0000000..e69de29 diff --git a/lts/ltsutils/__init__.py b/lts/ltsutils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lts/ltsutils/package_repo.py b/lts/ltsutils/package_repo.py new file mode 100644 index 0000000..7dcae87 --- /dev/null +++ b/lts/ltsutils/package_repo.py @@ -0,0 +1,52 @@ +import pathlib +from subprocess import PIPE, CalledProcessError +from .shell import Shell + +class PackageRepo: + '''Represents a package repository. Most methods are wrappers of git commands.''' + class UnknownCurrentBranchException(Exception): pass + class InvalidBranchException(Exception): pass + + def __init__(self, name, path): + self.name = name + self.path = pathlib.Path(path) + self.sh = Shell(self.path) + + def getNVR(self, commit='HEAD'): + with self.sh.popen(['git', 'show', '{}:{}.spec'.format(commit, self.name)], stdout=PIPE) as specfile: + nvr = self.sh.run('rpmspec --srpm -q --queryformat %{NVR} /dev/stdin', stdin=specfile.stdout) + return tuple(nvr.stdout.strip().split('-')) + + def checkoutBranch(self, branch, allow_remote=False): + # allow_remote=True allows checking out a new remote-tracking branch + if not allow_remote and not self.hasBranch(branch): + raise self.InvalidBranchException(branch) + self.sh.run_args(['git', 'checkout', branch], capture_output=False) + + def fastForwardBranch(self, old, new): + self.checkoutBranch(old) + if not self.hasBranch(new): + raise self.InvalidBranchException(new) + self.sh.run_args(['git', 'merge', '--ff-only', new], capture_output=False) + + def getActiveBranches(self): + toplvl = pathlib.Path(self.path) / '../..' + common = toplvl / 'projects/common' + active_branches = common / 'lts/active-branches' + with active_branches.open() as f: + return [line.rstrip() for line in f] + + def getCurrentBranch(self): + try: + head = self.sh.run('git symbolic-ref HEAD').stdout.strip() + except CalledProcessError: + raise self.UnknownCurrentBranchException + + refs_heads = 'refs/heads/' + assert head.startswith(refs_heads) + head = head[len(refs_heads):] + return head + + def hasBranch(self, branch): + p = self.sh.run_args(['git', 'rev-parse', 'refs/heads/'+branch], check=False) + return p.returncode == 0 diff --git a/lts/ltsutils/shell.py b/lts/ltsutils/shell.py new file mode 100644 index 0000000..f3aa16c --- /dev/null +++ b/lts/ltsutils/shell.py @@ -0,0 +1,27 @@ +import subprocess, shlex + +class Shell: + # Default options passed to subprocess.run. May be customized per-instance. + cwd = None + check = True + capture_output = True + text = True + + def __init__(self, cwd=None): + if cwd: self.cwd = cwd + + def run_args(self, args, **kwargs): + kwargs1 = { + 'check': self.check, + 'capture_output': self.capture_output, + 'text': self.text, + 'cwd': self.cwd + } + kwargs1.update(kwargs) + return subprocess.run(args, **kwargs1) + + def run(self, cmd, **kwargs): + return self.run_args(shlex.split(cmd), **kwargs) + + def popen(self, args, **kwargs): + return subprocess.Popen(args, cwd=self.cwd, **kwargs) diff --git a/lts/main.py b/lts/main.py new file mode 100644 index 0000000..a579f7d --- /dev/null +++ b/lts/main.py @@ -0,0 +1,91 @@ +#!/usr/bin/python +import sys,argparse + +from ltsutils.package_repo import PackageRepo + +def log(msg, **kwargs): + print(msg, file=sys.stderr) + +def init_parser(): + main = argparse.ArgumentParser() + main.add_argument('package_name', nargs=1) + subparsers = main.add_subparsers(dest='command', metavar='command', required=True) + + p = subparsers.add_parser('checkout-prev', + help='checkout previous branch') + p = subparsers.add_parser('is-same-version', + help='return true if package version is the same as the given branch') + p.add_argument('branch', nargs=1) + p = subparsers.add_parser('fast-forward', + help='fast-forward current branch to a newer branch') + p.add_argument('branch', nargs=1) + + p = subparsers.add_parser('sanity-check', + help='run sanity checks for data consistency') + + return main + +def checkout_prev(args, repo): + active_branches = repo.getActiveBranches() + current = repo.getCurrentBranch() + + i = active_branches.index(current) + if i == 0: + log('Already on oldest active branch.') + return False + + prev = active_branches[i-1] + repo.checkoutBranch(prev, allow_remote=True) + +def is_same_version(args, other): + current = repo.getCurrentBranch() + other = args.branch[0] + assert repo.hasBranch(other), 'Branch %s not found' % other + + v1, v2 = [repo.getNVR('refs/heads/'+b)[1] for b in (current, other)] + if v1 != v2: + log('Current version {} does not match version {} on branch {}'.format(v1, v2, other)) + return v1 == v2 + +def fast_forward(args, repo): + current = repo.getCurrentBranch() + newer = args.branch[0] + + log('Fast-forwarding {} to {}'.format(current, newer)) + repo.fastForwardBranch(current, newer) + +def sanity_check(args, repo): + ok = True + # HEAD must point to a branch + try: + current = repo.getCurrentBranch() + except PackageRepo.UnknownCurrentBranchException: + log('Unknown current branch. Has a branch been checked out?') + current = None + ok = False + + # active-branches file must not be empty + active_branches = repo.getActiveBranches() + if not len(active_branches): + log('No active branches defined. Is active-branches file empty?') + ok = False + # current branch must be an active branch + elif current and current not in active_branches: + log('%s is not an active LTS branch.' % current) + ok = False + + return ok + +if __name__=='__main__': + args = init_parser().parse_args() + repo = PackageRepo(args.package_name[0], '.') + + commands = { + 'checkout-prev': checkout_prev, + 'is-same-version': is_same_version, + 'fast-forward': fast_forward, + 'sanity-check': sanity_check, + } + ret = commands[args.command](args, repo) + if ret is not None: + exit(0 if ret else 1) diff --git a/lts/test/__init__.py b/lts/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lts/test/test_ltsutils.py b/lts/test/test_ltsutils.py new file mode 100644 index 0000000..1a11d31 --- /dev/null +++ b/lts/test/test_ltsutils.py @@ -0,0 +1,79 @@ +import unittest +import os, pathlib, tempfile, logging +import subprocess, functools + +from ltsutils.package_repo import PackageRepo +import ltsutils.shell + +run = functools.partial(subprocess.run, check=True) + +class LTSUtilsTestCase(unittest.TestCase): + toplvl = pathlib.Path('../../..') + packages = toplvl / 'packages' + pkgname = 'nano' + master_commit = '6947e7170435e179aaa86b156e53cae644ce1d73' + repo_url = 'https://github.com/clearlinux-pkgs/{}.git'.format(pkgname) + + def cloneOrExtractRepo(self): + tmpdir = pathlib.Path('/var/tmp/common-lts-test') + tmpdir.mkdir(mode=0o700, exist_ok=True) + tarball = tmpdir / '{}.tar.gz'.format(self.pkgname) + if tarball.exists(): + run(['tar', 'xf', tarball, '-C', self.workdir]) + else: + run(['git', 'clone', self.repo_url, self.workdir]) + run(['tar', 'czf', tarball, '-C', self.workdir, '.']) + + def setUp(self): + self._tmpdir = tempfile.TemporaryDirectory(prefix='test-{}-'.format(self.pkgname), dir=self.packages) + self.workdir = pathlib.Path(self._tmpdir.name) + self.cloneOrExtractRepo() + self.repo = PackageRepo(self.pkgname, self.workdir) + + self._sh = ltsutils.shell.Shell(self.workdir) + self._sh.capture_output = False + + def sh(self, cmd): + return self._sh.run(cmd) + + def sh_stdout(self, cmd): + return self._sh.run(cmd, capture_output=True).stdout.rstrip() + + def tearDown(self): + self._tmpdir.cleanup() + +class TestPackageRepo(LTSUtilsTestCase): + L1 = '3dcfa09f5217eedf6ec7539af7e243655d3abdb6' # 3.2-54 + L2 = 'b8243dd54e8feb16a11474f848b8735f5591cf12' # 3.2-55 + + def setUp(self): + super().setUp() + self.sh('git branch L1 %s' % self.L1) + self.sh('git branch L2 %s' % self.L2) + + def testGetNVR(self): + nvr = self.repo.getNVR(self.L2) + self.assertEqual(nvr, ('nano', '3.2', '55')) + + def testHasBranch(self): + self.assertTrue(self.repo.hasBranch('L2')) + self.assertFalse(self.repo.hasBranch('L3')) + + def testCheckoutBranch(self): + self.repo.checkoutBranch('L2') + self.assertEqual(self.sh_stdout('git rev-parse HEAD'), self.L2) + + self.assertRaises(PackageRepo.InvalidBranchException, self.repo.checkoutBranch, 'L3') + + def testFastForward(self): + self.repo.fastForwardBranch('L1', 'L2') + self.assertEqual(self.sh_stdout('git rev-parse L1'), self.L2) + self.assertEqual(self.sh_stdout('git rev-parse L2'), self.L2) + + def testGetCurrentBranch(self): + self.repo.checkoutBranch('L2') + b = self.repo.getCurrentBranch() + self.assertEqual(b, 'L2') + + self.sh('git checkout --detach L2') + self.assertRaises(PackageRepo.UnknownCurrentBranchException, self.repo.getCurrentBranch)