Re: [PR PATCH] [Updated] [RFC] enable tests in CI and introduce XBPS_CHECK_PKGS=full

[ericonr <ericonr@users.noreply.github.com>]

[-- Attachment #1: Type: text/plain, Size: 465 bytes --]

There is an updated pull request by ericonr against master on the void-packages repository

https://github.com/ericonr/void-packages ci
https://github.com/void-linux/void-packages/pull/26773

[RFC] enable tests in CI and introduce XBPS_CHECK_PKGS=full
This probably needs some work in templates to define a cheap `do_check` and a more expensive `do_all_checks` or whatever.

A patch file from https://github.com/void-linux/void-packages/pull/26773.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-ci-26773.patch --]
[-- Type: text/x-diff, Size: 10111 bytes --]

From 7551f96a4b8f2339d41a3f7589c0d6649041aab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Mon, 7 Dec 2020 20:15:42 -0300
Subject: [PATCH 1/5] xbps-src: add -K option for running the full testsuite.

Some packages have rather long test suites, with an option for a shorter
one. With this option, one can choose between shorter tests (-Q) or the
complete longer ones (-K).

This also allows CI to run only the shorter testsuites.

Make the appropriate changes to etc/default.conf and Manual.md as well.
---
 Manual.md         | 3 +++
 etc/defaults.conf | 2 ++
 xbps-src          | 7 +++++--
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Manual.md b/Manual.md
index 0a3c759f59b..aa7a196d4f4 100644
--- a/Manual.md
+++ b/Manual.md
@@ -364,6 +364,9 @@ rather than additional binary package names.
 
 - `CROSS_BUILD` Set if `xbps-src` is cross compiling a package.
 
+- `XBPS_CHECK_PKGS` Set if `xbps-src` is going to run tests for a package.
+Longer testsuites should only be run in `do_check()` if it is set to `full`.
+
 - `DESTDIR` Full path to the fake destdir used by the source pkg, set to
 `<masterdir>/destdir/${sourcepkg}-${version}`.
 
diff --git a/etc/defaults.conf b/etc/defaults.conf
index 6e297903fbd..88286deaa6f 100644
--- a/etc/defaults.conf
+++ b/etc/defaults.conf
@@ -71,8 +71,10 @@ XBPS_SUCMD="sudo /bin/sh -c"
 
 # [OPTIONAL]
 # Enable running the (optional) do_check() function of a package.
+# When set to 'full', will enable further testing for some packages.
 #
 #XBPS_CHECK_PKGS=yes
+#XBPS_CHECK_PKGS=full
 
 # [OPTIONAL]
 # Enable building -dbg subpackages with debugging symbols. Please note
diff --git a/xbps-src b/xbps-src
index 08da501f219..54016874010 100755
--- a/xbps-src
+++ b/xbps-src
@@ -205,6 +205,8 @@ $(print_cross_targets)
 
 -Q  Enable running the check stage.
 
+-K  Enable running the check stage with longer tests.
+
 -q  Suppress informational output of xbps-src (build output is still printed).
 
 -r  <repo>
@@ -381,7 +383,8 @@ while getopts "$XBPS_OPTSTRING" opt; do
         o) XBPS_ARG_PKG_OPTIONS="$OPTARG"; XBPS_OPTIONS+=" -o $OPTARG";;
         p) XBPS_ARG_PRINT_VARIABLES="$OPTARG"; XBPS_OPTIONS+=" -p $OPTARG";;
         q) XBPS_ARG_QUIET=1; XBPS_OPTIONS+=" -q";;
-        Q) XBPS_ARG_CHECK_PKGS=1; XBPS_OPTIONS+=" -Q";;
+        Q) XBPS_ARG_CHECK_PKGS=yes; XBPS_OPTIONS+=" -Q";;
+        K) XBPS_ARG_CHECK_PKGS=full; XBPS_OPTIONS+=" -K";;
         r) XBPS_ARG_ALT_REPOSITORY="$OPTARG"; XBPS_OPTIONS+=" -r $OPTARG";;
         t) XBPS_ARG_TEMP_MASTERDIR=1; XBPS_OPTIONS+=" -t -C";;
         V) echo "xbps-src-$XBPS_SRC_VERSION $(xbps-uhelper -V)" && exit 0;;
@@ -459,7 +462,6 @@ fi
 [ -n "$XBPS_ARG_TEMP_MASTERDIR" ] && XBPS_TEMP_MASTERDIR=1
 [ -n "$XBPS_ARG_BINPKG_EXISTS" ] && XBPS_BINPKG_EXISTS=1
 [ -n "$XBPS_ARG_USE_GIT_REVS" ] && XBPS_USE_GIT_REVS=1
-[ -n "$XBPS_ARG_CHECK_PKGS" ] && XBPS_CHECK_PKGS=1
 [ -n "$XBPS_ARG_DEBUG_PKGS" ] && XBPS_DEBUG_PKGS=1
 [ -n "$XBPS_ARG_SKIP_DEPS" ] && XBPS_SKIP_DEPS=1
 [ -n "$XBPS_ARG_KEEP_ALL" ] && XBPS_KEEP_ALL=1
@@ -467,6 +469,7 @@ fi
 [ -n "$XBPS_ARG_PRINT_VARIABLES" ] && XBPS_PRINT_VARIABLES="$XBPS_ARG_PRINT_VARIABLES"
 [ -n "$XBPS_ARG_ALT_REPOSITORY" ] && XBPS_ALT_REPOSITORY="$XBPS_ARG_ALT_REPOSITORY"
 [ -n "$XBPS_ARG_CROSS_BUILD" ] && XBPS_CROSS_BUILD="$XBPS_ARG_CROSS_BUILD"
+[ -n "$XBPS_ARG_CHECK_PKGS" ] && XBPS_CHECK_PKGS="$XBPS_ARG_CHECK_PKGS"
 [ -n "$XBPS_ARG_MAKEJOBS" ] && XBPS_MAKEJOBS="$XBPS_ARG_MAKEJOBS"
 
 export XBPS_BUILD_ONLY_ONE_PKG XBPS_SKIP_REMOTEREPOS XBPS_BUILD_FORCEMODE \

From 1e9acc22f1a6ea4be8ea272f41b82c75d6def913 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Wed, 2 Dec 2020 15:06:37 -0300
Subject: [PATCH 2/5] common/travis/fetch-xtools: make it work inside
 masterdir.

---
 common/travis/fetch-xtools.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/common/travis/fetch-xtools.sh b/common/travis/fetch-xtools.sh
index 7abf44bcc12..0dd9e6c532c 100755
--- a/common/travis/fetch-xtools.sh
+++ b/common/travis/fetch-xtools.sh
@@ -2,9 +2,18 @@
 #
 # fetch-xtools.sh
 
+TAR=tar
+command -v bsdtar >/dev/null && TAR=bsdtar
+URL="https://github.com/leahneukirchen/xtools/archive/master.tar.gz"
+FILE="xtools.tar.gz"
+
 mkdir -p /tmp/bin
 
 /bin/echo -e '\x1b[32mInstalling xtools...\x1b[0m'
-wget -q -O - https://github.com/leahneukirchen/xtools/archive/master.tar.gz | \
-	gunzip | tar x -C /tmp/bin --wildcards "xtools-master/x*" \
-	--strip-components=1 || exit 1
+if command -v wget >/dev/null; then
+	wget -q -O "$FILE" "$URL" || exit 1
+else
+	xbps-fetch -o "$FILE" "$URL" || exit 1
+fi
+
+$TAR xf "$FILE" -C /tmp/bin --strip-components=1 || exit 1

From 84aad24bd89a4ffb13d323aad942eba726b8e9e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Wed, 2 Dec 2020 15:30:48 -0300
Subject: [PATCH 3/5] common/travis/build.sh: add argument for running tests.

Also use nproc(1) directly and remove FTP_RETRIES.
---
 common/travis/build.sh | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/common/travis/build.sh b/common/travis/build.sh
index 1c45910ea69..f5cf9cffc06 100755
--- a/common/travis/build.sh
+++ b/common/travis/build.sh
@@ -6,17 +6,14 @@ if [ "$1" != "$2" ]; then
 	arch="-a $2"
 fi
 
-PKGS=$(/hostrepo/xbps-src sort-dependencies $(cat /tmp/templates))
-
-NPROCS=1
-if [ -r /proc/cpuinfo ]; then
-        NPROCS=$(grep ^proc /proc/cpuinfo|wc -l)
+if [ "$3" = 1 ]; then
+	test="-Q"
 fi
 
-export FTP_RETRIES=10
+PKGS=$(/hostrepo/xbps-src sort-dependencies $(cat /tmp/templates))
 
 for pkg in ${PKGS}; do
-	/hostrepo/xbps-src -j$NPROCS -H "$HOME"/hostdir $arch pkg "$pkg"
+	/hostrepo/xbps-src -j$(nproc) -H "$HOME"/hostdir $arch $test pkg "$pkg"
 	[ $? -eq 1 ] && exit 1
 done
 

From ccf0545319ff51764d785988b36e7080b5e42c6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Sat, 28 Nov 2020 15:47:04 -0300
Subject: [PATCH 4/5] .github/workflows: run tests on CI for native builds.

Also:
- fix "list contents" step by adding a fetch-xtools step.
- remove git from masterdir to not interfere with builds (thanks @sgn)
---
 .github/workflows/build.yaml | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 8535d3b24b1..be2fd7969db 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -34,19 +34,20 @@ jobs:
         PATH: '/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin:/usr/local/bin:/tmp/bin'
         ARCH: '${{ matrix.config.arch }}'
         BOOTSTRAP: '${{ matrix.config.bootstrap }}'
+        TEST: '${{ matrix.config.test }}'
         HOSTREPO: /hostrepo
 
     strategy:
       fail-fast: false
       matrix:
         config:
-          - { arch: x86_64, bootstrap: x86_64 }
-          - { arch: i686, bootstrap: i686 }
-          - { arch: aarch64, bootstrap: x86_64 }
-          - { arch: armv7l, bootstrap: x86_64 }
-          - { arch: x86_64-musl, bootstrap: x86_64-musl }
-          - { arch: armv6l-musl, bootstrap: x86_64-musl }
-          - { arch: aarch64-musl, bootstrap: x86_64-musl }
+          - { arch: x86_64, bootstrap: x86_64, test: 1 }
+          - { arch: i686, bootstrap: i686, test: 1 }
+          - { arch: aarch64, bootstrap: x86_64, test: 0 }
+          - { arch: armv7l, bootstrap: x86_64, test: 0 }
+          - { arch: x86_64-musl, bootstrap: x86_64-musl, test: 1 }
+          - { arch: armv6l-musl, bootstrap: x86_64-musl, test: 0 }
+          - { arch: aarch64-musl, bootstrap: x86_64-musl, test: 0 }
 
     steps:
       - name: Prepare container
@@ -56,7 +57,7 @@ jobs:
           # Upgrade again (in case there was a xbps update)
           xbps-install -yu
           # Install git
-          xbps-install -y git xtools
+          xbps-install -y git
 
       - uses: actions/checkout@v1
         with:
@@ -67,13 +68,16 @@ jobs:
       - run: common/travis/prepare.sh
       - run: common/travis/fetch_upstream.sh
       - run: common/travis/changed_templates.sh
+      - run: common/travis/fetch-xtools.sh
+      - name: Remove git after checkout to not interfere with build
+        run: xbps-remove -yR git
 
-      - name: Build packages
+      - name: Build and check packages
         run: |
           (
           here="$(pwd)"
           cd /
-          "$here/common/travis/build.sh" "$BOOTSTRAP" "$ARCH"
+          "$here/common/travis/build.sh" "$BOOTSTRAP" "$ARCH" "$TEST"
           )
 
       - name: Show files

From d3942ade080dd35e2caee64d5133f6b7cea00b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Mon, 7 Dec 2020 20:15:12 -0300
Subject: [PATCH 5/5] libaom: split tests by duration.

---
 srcpkgs/libaom/template | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/srcpkgs/libaom/template b/srcpkgs/libaom/template
index 93f1983edfc..657301c7309 100644
--- a/srcpkgs/libaom/template
+++ b/srcpkgs/libaom/template
@@ -1,6 +1,6 @@
 # Template file for 'libaom'
 pkgname=libaom
-version=2.0.0
+version=2.0.1
 revision=1
 create_wrksrc=yes
 build_style=cmake
@@ -12,7 +12,7 @@ license="BSD-2-Clause"
 homepage="https://aomedia.org/"
 changelog="https://aomedia.googlesource.com/aom/+/master/CHANGELOG"
 distfiles="https://aomedia.googlesource.com/aom/+archive/v${version}.tar.gz"
-checksum=@4e8a74ed7e05d5534ab0b72735fa670ce1a1e757bc4a777f07fdabf8629fced1
+checksum=@b89acd3304531a9f6c99eba13e1de66605893ff8d08abc8c6e0586481a90033a
 
 # aom segfaults with default musl stack size
 LDFLAGS="-Wl,-z,stack-size=2097152"
@@ -29,14 +29,18 @@ if [ -z "$XBPS_CHECK_PKGS" ]; then
 fi
 
 pre_check() {
-	mkdir -p /host/libaom-test-data
 	export LIBAOM_TEST_DATA_PATH=/host/libaom-test-data
+	mkdir -p $LIBAOM_TEST_DATA_PATH
 }
 
 do_check() {
 	cd build
-	make runtests ${makejobs}
+	make testdata ${makejobs}
 	../test/examples.sh --bin-path examples
+
+	if [ "$XBPS_CHECK_PKGS" = full ]; then
+		make runtests ${makejobs}
+	fi
 }
 
 post_install() {