From a7e3cdb63b7e681158db4f57c75204c7d0cccc53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89rico=20Nogueira?= Date: Wed, 23 Jun 2021 13:39:19 -0300 Subject: [PATCH 1/2] common/scripts: import xbps-cycles. From https://github.com/ahesford/xbps-cycles, license is compatible with void-packages. Will be run in CI, so it should live in the same repository. --- common/scripts/README.xbps-cycles.md | 23 ++++++ common/scripts/xbps-cycles.py | 102 +++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 common/scripts/README.xbps-cycles.md create mode 100755 common/scripts/xbps-cycles.py diff --git a/common/scripts/README.xbps-cycles.md b/common/scripts/README.xbps-cycles.md new file mode 100644 index 000000000000..458f3787278b --- /dev/null +++ b/common/scripts/README.xbps-cycles.md @@ -0,0 +1,23 @@ +# Cycle detector for void-packages + +This script enumerates dependencies for packages in a +[void-packages repository](https://github.com/void-linux/void-packages) +and identifies build-time dependency cycles. It is based on +[earlier scripts](https://gist.github.com/Chocimier/de76441493ec7775c201dac0bb03ced5) +provided by the Void maintainer @Chocimier. The key differences are +- No intermediate files are created +- Dependency enumeration is parallelized by default +- Output provides a more illustrative view of cycles + +For command syntax, run `xbps-cycles.py -h`. Often, it may be sufficient to run +`xbps-cycles.py` with no arguments. By default, the script will look for a +repository at `$XBPS_DISTDIR`; if that variable is not defined, the current +directory is used instead. To override this behavior, use the `-d` option to +provide the path to your desired void-packages clone. + +The standard behavior will be to spawn multiple processes, one per CPU, to +enumerate package dependencies. This is by far the most time-consuming part of +the execution. To override the degree of parallelism, use the `-j` option. + +Failures should be harmless but, at this early stage, unlikely to be pretty or +even helpful. diff --git a/common/scripts/xbps-cycles.py b/common/scripts/xbps-cycles.py new file mode 100755 index 000000000000..24ef17156336 --- /dev/null +++ b/common/scripts/xbps-cycles.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import os +import sys +import glob +import subprocess +import multiprocessing + +from argparse import ArgumentParser + +import networkx as nx + + +def enum_depends(pkg, xbpsdir): + ''' + Return a pair (pkg, [dependencies]), where [dependencies] is the list + of dependencies for the given package pkg. The argument xbpsdir should + be a path to a void-packages repository. Dependencies will be + determined by invoking + + /xbps-src show-build-deps + + If the return code of this call nonzero, a message will be printed but + the package will treated as if it has no dependencies. + ''' + cmd = [os.path.join(xbpsdir, 'xbps-src'), 'show-build-deps', pkg] + + try: + deps = subprocess.check_output(cmd) + except subprocess.CalledProcessError as err: + print('xbps-src failed to find dependencies for package', pkg) + deps = [ ] + else: + deps = [d for d in deps.decode('utf-8').split('\n') if d] + + return pkg, deps + + +def find_cycles(depmap, xbpsdir): + ''' + For a map depmap: package -> [dependencies], construct a directed graph + and identify any cycles therein. + + The argument xbpsdir should be a path to the root of a void-packages + repository. All package names in depmap will be appended to the path + /srcpkgs and reduced with os.path.realpath to coalesce + subpackages. + ''' + G = nx.DiGraph() + + for i, deps in depmap.items(): + path = os.path.join(xbpsdir, 'srcpkgs', i) + i = os.path.basename(os.path.realpath(path)) + + for j in deps: + path = os.path.join(xbpsdir, 'srcpkgs', j.strip()) + j = os.path.basename(os.path.realpath(path)) + G.add_edge(i, j) + + for c in nx.strongly_connected_components(G): + if len(c) < 2: continue + pkgs = nx.to_dict_of_lists(G, c) + + p = next(iter(pkgs.keys())) + cycles = [ ] + while True: + cycles.append(p) + + # Cycle is complete when package is not in map + try: deps = pkgs.pop(p) + except KeyError: break + + # Any of the dependencies here contributes to a cycle + p = deps[0] + if len(deps) > 1: + print('Mulitpath: {} -> {}, choosing first'.format(p, deps)) + + if cycles: + print('Cycle: ' + ' -> '.join(cycles) + '\n') + + +if __name__ == '__main__': + parser = ArgumentParser(description='Cycle detector for xbps-src') + parser.add_argument('-j', '--jobs', default=None, + type=int, help='Number of parallel jobs') + parser.add_argument('-d', '--directory', + default=None, help='Path to void-packages repo') + + args = parser.parse_args() + + if not args.directory: + try: args.directory = os.environ['XBPS_DISTDIR'] + except KeyError: args.directory = '.' + + pool = multiprocessing.Pool(processes = args.jobs) + + pattern = os.path.join(args.directory, 'srcpkgs', '*') + depmap = dict(pool.starmap(enum_depends, + ((os.path.basename(g), args.directory) + for g in glob.iglob(pattern)))) + + find_cycles(depmap, args.directory) From e9a46f7e3913456c2bc4ad840e0623c6f028bbbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89rico=20Nogueira?= Date: Wed, 23 Jun 2021 13:45:23 -0300 Subject: [PATCH 2/2] .github/workflows: run xbps-cycles daily. Should help in catching cyclic dependencies early. Rename lockthreads.yml to include all scheduled CI tasks. --- .github/workflows/{lockthreads.yml => cron.yml} | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename .github/workflows/{lockthreads.yml => cron.yml} (61%) diff --git a/.github/workflows/lockthreads.yml b/.github/workflows/cron.yml similarity index 61% rename from .github/workflows/lockthreads.yml rename to .github/workflows/cron.yml index f3ec106a6e6c..c87446743fe5 100644 --- a/.github/workflows/lockthreads.yml +++ b/.github/workflows/cron.yml @@ -1,4 +1,4 @@ -name: 'Lock threads' +name: 'Scheduled tasks' on: schedule: @@ -13,3 +13,8 @@ jobs: github-token: ${{ github.token }} pr-lock-inactive-days: '90' process-only: 'prs' + cycles: + runs-on: ubuntu-latest + steps: + - run: apt-get install -y python3-networkx + - run: common/scripts/xbps-cycles.py