Github messages for voidlinux
 help / color / mirror / Atom feed
* [PR PATCH] Check file conflicts in x86_64 build job
@ 2022-04-26 20:55 Chocimier
  2022-05-02 14:13 ` [PR REVIEW] " classabbyamp
                   ` (16 more replies)
  0 siblings, 17 replies; 18+ messages in thread
From: Chocimier @ 2022-04-26 20:55 UTC (permalink / raw)
  To: ml

[-- Attachment #1: Type: text/plain, Size: 419 bytes --]

There is a new pull request by Chocimier against master on the void-packages repository

https://github.com/Chocimier/void-packages-org lint-conflicts
https://github.com/void-linux/void-packages/pull/36876

Check file conflicts in x86_64 build job
Report packages installing same file and not marked with conflicts or replaces.

A patch file from https://github.com/void-linux/void-packages/pull/36876.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-lint-conflicts-36876.patch --]
[-- Type: text/x-diff, Size: 7721 bytes --]

From 8ccedb3d2f633a450eaf35dc42fe38bc19ee65b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Tue, 26 Apr 2022 20:05:50 +0200
Subject: [PATCH 1/3] common/travis: use patched xtools

---
 common/travis/fetch-xtools.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/travis/fetch-xtools.sh b/common/travis/fetch-xtools.sh
index 0dd9e6c532c0..2006774a1bc5 100755
--- a/common/travis/fetch-xtools.sh
+++ b/common/travis/fetch-xtools.sh
@@ -4,7 +4,7 @@
 
 TAR=tar
 command -v bsdtar >/dev/null && TAR=bsdtar
-URL="https://github.com/leahneukirchen/xtools/archive/master.tar.gz"
+URL="https://github.com/Chocimier/xtools/archive/master.tar.gz"
 FILE="xtools.tar.gz"
 
 mkdir -p /tmp/bin

From ba5b1d593e15420f21fca741ec83cdef928e24e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Mon, 25 Apr 2022 23:02:57 +0200
Subject: [PATCH 2/3] common/: add script finding file conflicts

---
 common/scripts/lint-conflicts | 187 ++++++++++++++++++++++++++++++++++
 1 file changed, 187 insertions(+)
 create mode 100755 common/scripts/lint-conflicts

diff --git a/common/scripts/lint-conflicts b/common/scripts/lint-conflicts
new file mode 100755
index 000000000000..2fd5c9eea012
--- /dev/null
+++ b/common/scripts/lint-conflicts
@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+
+# Report packages installing same file and not marked with
+# conflicts or replaces.
+# If file /tmp/templates is present, find conflicts between packages
+# listed there one per line, including their subpackages,
+# and packages indexed in xlocate. The convention is taken from CI.
+# If the file is not present, find conflicts between packages indexed
+# in xlocate.
+
+export XBPS_TARGET_ARCH="$2" XBPS_DISTDIR=/hostrepo
+
+declare -A newly_built conflicts_cache providers_cache pairs owners
+repositories="--repository=$HOME/hostdir/binpkgs --repository=$HOME/hostdir/binpkgs/nonfree"
+rv=0
+
+template_exists() {
+	[ -e "srcpkgs/$1/template" ]
+}
+
+partial_check() {
+	[ -s /tmp/templates ]
+}
+
+providers_of() {
+	# print the pkgname and packages that `provides` it
+	local pkgname=$1
+	local line provider_pkgver provided_pkgver provider_pkgname provided_pkgname
+	if [ "${providers_cache[$pkgname]}" = '' ]; then
+		declare -A providers
+		providers[$pkgname]=$pkgname
+		while read -r line; do
+			line=${line%%'('*}
+			provider_pkgver=${line%': '*}
+			provided_pkgver=${line#*': '}
+			provider_pkgname=${provider_pkgver%-*}
+			provided_pkgname=${provided_pkgver%-*}
+			# comes from $(xbps-query -s $pkgname), so $pkgname can be substring
+			if [ "$provided_pkgname" = "$pkgname" ]; then
+				providers[$provider_pkgname]=$provider_pkgname
+			fi
+		done < <(xbps-query $repositories -p provides -R -s "$pkgname")
+		# leading space ensures ${[]} != ''
+		providers_cache[$pkgname]=" ${providers[*]}"
+	fi
+	echo ${providers_cache[$pkgname]}
+}
+
+conflicts_of() {
+	# print list of packages that are _marked_ as confliting with given one
+	local pkgname=$1
+	local in_conflict provider
+	if [ "${conflicts_cache[$pkgname]}" = '' ]; then
+		local -A all
+		while read -r in_conflict; do
+			in_conflict=${in_conflict%'<'*}
+			in_conflict=${in_conflict%'>'*}
+			providers_of "$in_conflict" > /dev/null # executing in same process to fill cache
+			for provider in $(providers_of "$in_conflict"); do
+				all[$provider]=$provider
+			done
+		done < <(xbps-query $repositories -p conflicts,replaces -R "$pkgname")
+		# leading space ensures ${[]} != ''
+		conflicts_cache[$pkgname]=" ${all[*]}"
+	fi
+	echo ${conflicts_cache[$pkgname]}
+}
+
+conflict_between() {
+	# exit successfully if packages are _marked_ as conflicting
+	if [ "$1" = "$2" ]; then
+		return 0 # can't install twice ;)
+	fi
+	conflicts_of "$1" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$1") " in
+		*" $2 "*) return 0
+	esac
+	conflicts_of "$2" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$2") " in
+		*" $1 "*) return 0
+	esac
+	return 1
+}
+
+list_newly_built_files() {
+	# print one line per file in newly build packages
+	# each line contains pkgname and file path
+	local pkg subpkg file
+	while read -r pkg; do
+		for subpkg in $(xsubpkg "$pkg"); do
+			while read -r file; do
+				echo "$subpkg" "$file"
+			done < <(xbps-query $repositories -i -f "$subpkg" | sed s'/ -> .*//')
+		done
+	done < /tmp/templates
+}
+
+list_interesting_files() {
+	# list files potentially contained in more than one package
+	# each line contains pkgname and file path
+	if partial_check; then
+		list_newly_built_files
+	else
+		xlocate / | sed s'/ -> .*//' | grep -F -f <(xlocate / | cut -f 2- | sed s'/ -> .*//' | sort | uniq -d)
+	fi
+}
+
+group_by_file_full() {
+	# create associative array `owners` mapping file to list of packages
+	# for packages potentially conflicting with newly built ones
+	local pkgver file pkgname
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(list_interesting_files)
+}
+
+group_by_file_partial() {
+	# create associative array `owners` mapping file to list of packages
+	# for all packages in xlocate
+	local pkgname file pkgver
+	## newly build packages
+	while read -r pkgname file; do
+		owners[$file]+=" $pkgname"
+		newly_built[$pkgname]=$pkgname
+	done < <(list_newly_built_files)
+	## rest of repository
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if [ -z "${newly_built[$pkgname]}" ] && template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(xlocate / | sed s'/ -> .*//' | grep -F -f <(list_newly_built_files | cut -d ' ' -f 2-))
+}
+
+group_by_pair() {
+	# find package pairs owning same file and not marked as conflicting
+	local pkg file a b
+	while read -r pkg file; do
+		for a in ${owners[$file]}; do
+			for b in ${owners[$file]}; do
+				if ! [ "$a" "<" "$b" ]; then
+					continue
+				fi
+				if partial_check && [ -z "${newly_built[$a]}" ] && [ -z "${newly_built[$b]}" ]; then
+					continue
+				fi
+				if ! conflict_between "$a" "$b"; then
+					unset pair_files
+					local -A pair_files
+					eval "${pairs["$a $b"]}"
+					pair_files[$file]="$file"
+					pairs["$a $b"]="${pair_files[@]@A}"
+				fi
+			done
+		done
+	done < <(list_interesting_files)
+}
+
+print_out() {
+	local pair file
+	if [ 0 = "${#pairs[@]}" ]; then
+		echo 1>&2 "No conflicts found"
+		exit 0
+	fi
+	while read -r pair; do
+		rv=1
+		echo "${pair% *} and ${pair#* } conflict for"
+		unset pair_files
+		eval "${pairs[$pair]}"
+		for file in "${pair_files[@]}"; do
+			echo "  $file"
+		done | sort
+	done < <(printf '%s\n' "${!pairs[@]}" | sort)
+}
+
+if partial_check; then
+	group_by_file_partial
+else
+	group_by_file_full
+fi
+group_by_pair
+print_out
+
+exit $rv

From a93c62ce18158fc1259e171b6f54dd59c18d58d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Tue, 26 Apr 2022 17:13:30 +0200
Subject: [PATCH 3/3] .github: check file conflicts in x86_64 build job

---
 .github/workflows/build.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 2ae7a3e728fa..f72d6f6c2c43 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -94,6 +94,12 @@ jobs:
           "$here/common/travis/show_files.sh" "$BOOTSTRAP" "$ARCH"
           )
 
+      - name: Check file conflicts
+        if: matrix.config.arch == 'x86_64' # the arch indexed in xlocate
+        run: |
+          xlocate -S &&
+          common/scripts/lint-conflicts
+
       - name: Verify repository state
         run: |
           (

^ permalink raw reply	[flat|nested] 18+ messages in thread
end of thread, other threads:[~2022-05-05 19:29 UTC | newest]

Thread overview: 18+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-04-26 20:55 [PR PATCH] Check file conflicts in x86_64 build job Chocimier
2022-05-02 14:13 ` [PR REVIEW] " classabbyamp
2022-05-02 14:13 ` classabbyamp
2022-05-02 14:13 ` classabbyamp
2022-05-02 14:13 ` classabbyamp
2022-05-02 14:13 ` classabbyamp
2022-05-04 21:02 ` [PR PATCH] [Updated] " Chocimier
2022-05-04 21:03 ` [PR REVIEW] " Chocimier
2022-05-04 21:03 ` Chocimier
2022-05-04 21:08 ` Chocimier
2022-05-04 21:11 ` [PR PATCH] [Updated] " Chocimier
2022-05-04 21:12 ` Chocimier
2022-05-04 21:12 ` [PR REVIEW] " Chocimier
2022-05-05  0:01 ` classabbyamp
2022-05-05  1:52 ` 0x5c
2022-05-05 16:34 ` Chocimier
2022-05-05 17:10 ` 0x5c
2022-05-05 19:29 ` [PR PATCH] [Merged]: " Chocimier

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).