Re: [PR PATCH] [Updated] [RFC] enable tests in CI

[ericonr <ericonr@users.noreply.github.com>]

[-- Attachment #1: Type: text/plain, Size: 430 bytes --]

There is an updated pull request by ericonr against master on the void-packages repository

https://github.com/ericonr/void-packages ci
https://github.com/void-linux/void-packages/pull/26773

[RFC] enable tests in CI
This probably needs some work in templates to define a cheap `do_check` and a more expensive `do_all_checks` or whatever.

A patch file from https://github.com/void-linux/void-packages/pull/26773.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-ci-26773.patch --]
[-- Type: text/x-diff, Size: 8501 bytes --]

From ab6e8628f1c39cc3a9eeac6765dbd9de2aa993f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Mon, 7 Dec 2020 20:15:42 -0300
Subject: [PATCH 1/6] xbps-src: add -K option for running the full testsuite.

Some packages have rather long test suites, with an option for a shorter
one. With this option, one can choose between shorter tests (-Q) or the
complete longer ones (-K).

This also allows CI to run only the shorter testsuites.

Make the appropriate changes to etc/default.conf and Manual.md as well.
---
 Manual.md         | 3 +++
 etc/defaults.conf | 2 ++
 xbps-src          | 1 +
 3 files changed, 6 insertions(+)

diff --git a/Manual.md b/Manual.md
index 7647f957129..4b0d678a757 100644
--- a/Manual.md
+++ b/Manual.md
@@ -364,6 +364,9 @@ rather than additional binary package names.
 
 - `CROSS_BUILD` Set if `xbps-src` is cross compiling a package.
 
+- `XBPS_CHECK_PKGS` Set if `xbps-src` is going to run tests for a package.
+Longer testsuites should only be run in `do_check()` if it is set to `full`.
+
 - `DESTDIR` Full path to the fake destdir used by the source pkg, set to
 `<masterdir>/destdir/${sourcepkg}-${version}`.
 
diff --git a/etc/defaults.conf b/etc/defaults.conf
index 6e297903fbd..88286deaa6f 100644
--- a/etc/defaults.conf
+++ b/etc/defaults.conf
@@ -71,8 +71,10 @@ XBPS_SUCMD="sudo /bin/sh -c"
 
 # [OPTIONAL]
 # Enable running the (optional) do_check() function of a package.
+# When set to 'full', will enable further testing for some packages.
 #
 #XBPS_CHECK_PKGS=yes
+#XBPS_CHECK_PKGS=full
 
 # [OPTIONAL]
 # Enable building -dbg subpackages with debugging symbols. Please note
diff --git a/xbps-src b/xbps-src
index 08da501f219..bb06305d4ac 100755
--- a/xbps-src
+++ b/xbps-src
@@ -382,6 +382,7 @@ while getopts "$XBPS_OPTSTRING" opt; do
         p) XBPS_ARG_PRINT_VARIABLES="$OPTARG"; XBPS_OPTIONS+=" -p $OPTARG";;
         q) XBPS_ARG_QUIET=1; XBPS_OPTIONS+=" -q";;
         Q) XBPS_ARG_CHECK_PKGS=1; XBPS_OPTIONS+=" -Q";;
+        K) XBPS_ARG_CHECK_PKGS=full; XBPS_OPTIONS+=" -K";;
         r) XBPS_ARG_ALT_REPOSITORY="$OPTARG"; XBPS_OPTIONS+=" -r $OPTARG";;
         t) XBPS_ARG_TEMP_MASTERDIR=1; XBPS_OPTIONS+=" -t -C";;
         V) echo "xbps-src-$XBPS_SRC_VERSION $(xbps-uhelper -V)" && exit 0;;

From 2724fb51cf6143345cd01591c4e18ac7f567111d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Wed, 2 Dec 2020 15:06:37 -0300
Subject: [PATCH 2/6] common/travis/fetch-xtools: make it work inside
 masterdir.

---
 common/travis/fetch-xtools.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/common/travis/fetch-xtools.sh b/common/travis/fetch-xtools.sh
index 7abf44bcc12..0dd9e6c532c 100755
--- a/common/travis/fetch-xtools.sh
+++ b/common/travis/fetch-xtools.sh
@@ -2,9 +2,18 @@
 #
 # fetch-xtools.sh
 
+TAR=tar
+command -v bsdtar >/dev/null && TAR=bsdtar
+URL="https://github.com/leahneukirchen/xtools/archive/master.tar.gz"
+FILE="xtools.tar.gz"
+
 mkdir -p /tmp/bin
 
 /bin/echo -e '\x1b[32mInstalling xtools...\x1b[0m'
-wget -q -O - https://github.com/leahneukirchen/xtools/archive/master.tar.gz | \
-	gunzip | tar x -C /tmp/bin --wildcards "xtools-master/x*" \
-	--strip-components=1 || exit 1
+if command -v wget >/dev/null; then
+	wget -q -O "$FILE" "$URL" || exit 1
+else
+	xbps-fetch -o "$FILE" "$URL" || exit 1
+fi
+
+$TAR xf "$FILE" -C /tmp/bin --strip-components=1 || exit 1

From d4bde101ade0a35729b72bde56a26ba48aaa19f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Wed, 2 Dec 2020 15:30:48 -0300
Subject: [PATCH 3/6] common/travis/build.sh: don't clean masterdir for native
 builds.

This allows tests to be run in a separate step without rebuilding
everything.

Also use nproc(1) directly.
---
 common/travis/build.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/common/travis/build.sh b/common/travis/build.sh
index 1c45910ea69..6530afd906d 100755
--- a/common/travis/build.sh
+++ b/common/travis/build.sh
@@ -6,17 +6,17 @@ if [ "$1" != "$2" ]; then
 	arch="-a $2"
 fi
 
-PKGS=$(/hostrepo/xbps-src sort-dependencies $(cat /tmp/templates))
-
-NPROCS=1
-if [ -r /proc/cpuinfo ]; then
-        NPROCS=$(grep ^proc /proc/cpuinfo|wc -l)
+# keep builddir so tests can be run without rebuilding everything
+if [ "$3" = 1 ]; then
+	test="-C"
 fi
 
+PKGS=$(/hostrepo/xbps-src sort-dependencies $(cat /tmp/templates))
+
 export FTP_RETRIES=10
 
 for pkg in ${PKGS}; do
-	/hostrepo/xbps-src -j$NPROCS -H "$HOME"/hostdir $arch pkg "$pkg"
+	/hostrepo/xbps-src -j$(nproc) -H "$HOME"/hostdir $arch $test pkg "$pkg"
 	[ $? -eq 1 ] && exit 1
 done
 

From 1bea016bb0cbc6b07ced8dcb25bc04485295e078 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Wed, 2 Dec 2020 15:31:32 -0300
Subject: [PATCH 4/6] common/travis/test.sh: create file.

---
 common/travis/test.sh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100755 common/travis/test.sh

diff --git a/common/travis/test.sh b/common/travis/test.sh
new file mode 100755
index 00000000000..a50d4274b26
--- /dev/null
+++ b/common/travis/test.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# test.sh
+
+if [ "$1" != 1 ]; then
+	echo "Skipping tests..."
+	exit 0
+fi
+
+PKGS=$(cat /tmp/templates)
+
+export FTP_RETRIES=10
+
+for pkg in ${PKGS}; do
+	/hostrepo/xbps-src -j$(nproc) -H "$HOME"/hostdir -Q check "$pkg"
+	[ $? -eq 1 ] && exit 1
+done
+
+exit 0

From 8d2dc1b3fe2229f7336a93648e9d74b49e4774f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Sat, 28 Nov 2020 15:47:04 -0300
Subject: [PATCH 5/6] .github/workflows: run tests on CI for native builds.

Also fix "list contents" step by adding a fetch-xtools step.
---
 .github/workflows/build.yaml | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 33186037f8d..79d4368d77c 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -34,19 +34,20 @@ jobs:
         PATH: '/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin:/usr/local/bin:/tmp/bin'
         ARCH: '${{ matrix.config.arch }}'
         BOOTSTRAP: '${{ matrix.config.bootstrap }}'
+        TEST: '${{ matrix.config.test }}'
         HOSTREPO: /hostrepo
 
     strategy:
       fail-fast: false
       matrix:
         config:
-          - { arch: x86_64, bootstrap: x86_64 }
-          - { arch: i686, bootstrap: i686 }
-          - { arch: aarch64, bootstrap: x86_64 }
-          - { arch: armv7l, bootstrap: x86_64 }
-          - { arch: x86_64-musl, bootstrap: x86_64-musl }
-          - { arch: armv6l-musl, bootstrap: x86_64-musl }
-          - { arch: aarch64-musl, bootstrap: x86_64-musl }
+          - { arch: x86_64, bootstrap: x86_64, test: 1 }
+          - { arch: i686, bootstrap: i686, test: 1 }
+          - { arch: aarch64, bootstrap: x86_64, test: 0 }
+          - { arch: armv7l, bootstrap: x86_64, test: 0 }
+          - { arch: x86_64-musl, bootstrap: x86_64-musl, test: 1 }
+          - { arch: armv6l-musl, bootstrap: x86_64-musl, test: 0 }
+          - { arch: aarch64-musl, bootstrap: x86_64-musl, test: 0 }
 
     steps:
       - name: Prepare container
@@ -73,9 +74,18 @@ jobs:
           (
           here="$(pwd)"
           cd /
-          "$here/common/travis/build.sh" "$BOOTSTRAP" "$ARCH"
+          "$here/common/travis/build.sh" "$BOOTSTRAP" "$ARCH" "$TEST"
           )
 
+      - name: Check packages
+        run: |
+          (
+          here="$(pwd)"
+          cd /
+          "$here/common/travis/test.sh" "$TEST"
+          )
+
+      - run: common/travis/fetch-xtools.sh
       - name: Show files
         run: |
           (

From 947ab58b287f604a4e8aabee6c2c7ab60d2aa59d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89rico=20Rolim?= <erico.erc@gmail.com>
Date: Mon, 7 Dec 2020 20:15:12 -0300
Subject: [PATCH 6/6] libaom: split tests by duration.

---
 srcpkgs/libaom/template | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/srcpkgs/libaom/template b/srcpkgs/libaom/template
index 93f1983edfc..0ed1adefeb6 100644
--- a/srcpkgs/libaom/template
+++ b/srcpkgs/libaom/template
@@ -35,8 +35,11 @@ pre_check() {
 
 do_check() {
 	cd build
-	make runtests ${makejobs}
 	../test/examples.sh --bin-path examples
+
+	if [ "$XBPS_CHECK_PKGS" = full ]; then
+		make runtests ${makejobs}
+	fi
 }
 
 post_install() {