[PR PATCH] Check file conflicts in x86_64 build job

[PR PATCH] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 419 bytes --]

There is a new pull request by Chocimier against master on the void-packages repository

https://github.com/Chocimier/void-packages-org lint-conflicts
https://github.com/void-linux/void-packages/pull/36876

Check file conflicts in x86_64 build job
Report packages installing same file and not marked with conflicts or replaces.

A patch file from https://github.com/void-linux/void-packages/pull/36876.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-lint-conflicts-36876.patch --]
[-- Type: text/x-diff, Size: 7721 bytes --]

From 8ccedb3d2f633a450eaf35dc42fe38bc19ee65b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Tue, 26 Apr 2022 20:05:50 +0200
Subject: [PATCH 1/3] common/travis: use patched xtools

---
 common/travis/fetch-xtools.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/travis/fetch-xtools.sh b/common/travis/fetch-xtools.sh
index 0dd9e6c532c0..2006774a1bc5 100755
--- a/common/travis/fetch-xtools.sh
+++ b/common/travis/fetch-xtools.sh
@@ -4,7 +4,7 @@
 
 TAR=tar
 command -v bsdtar >/dev/null && TAR=bsdtar
-URL="https://github.com/leahneukirchen/xtools/archive/master.tar.gz"
+URL="https://github.com/Chocimier/xtools/archive/master.tar.gz"
 FILE="xtools.tar.gz"
 
 mkdir -p /tmp/bin

From ba5b1d593e15420f21fca741ec83cdef928e24e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Mon, 25 Apr 2022 23:02:57 +0200
Subject: [PATCH 2/3] common/: add script finding file conflicts

---
 common/scripts/lint-conflicts | 187 ++++++++++++++++++++++++++++++++++
 1 file changed, 187 insertions(+)
 create mode 100755 common/scripts/lint-conflicts

diff --git a/common/scripts/lint-conflicts b/common/scripts/lint-conflicts
new file mode 100755
index 000000000000..2fd5c9eea012
--- /dev/null
+++ b/common/scripts/lint-conflicts
@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+
+# Report packages installing same file and not marked with
+# conflicts or replaces.
+# If file /tmp/templates is present, find conflicts between packages
+# listed there one per line, including their subpackages,
+# and packages indexed in xlocate. The convention is taken from CI.
+# If the file is not present, find conflicts between packages indexed
+# in xlocate.
+
+export XBPS_TARGET_ARCH="$2" XBPS_DISTDIR=/hostrepo
+
+declare -A newly_built conflicts_cache providers_cache pairs owners
+repositories="--repository=$HOME/hostdir/binpkgs --repository=$HOME/hostdir/binpkgs/nonfree"
+rv=0
+
+template_exists() {
+	[ -e "srcpkgs/$1/template" ]
+}
+
+partial_check() {
+	[ -s /tmp/templates ]
+}
+
+providers_of() {
+	# print the pkgname and packages that `provides` it
+	local pkgname=$1
+	local line provider_pkgver provided_pkgver provider_pkgname provided_pkgname
+	if [ "${providers_cache[$pkgname]}" = '' ]; then
+		declare -A providers
+		providers[$pkgname]=$pkgname
+		while read -r line; do
+			line=${line%%'('*}
+			provider_pkgver=${line%': '*}
+			provided_pkgver=${line#*': '}
+			provider_pkgname=${provider_pkgver%-*}
+			provided_pkgname=${provided_pkgver%-*}
+			# comes from $(xbps-query -s $pkgname), so $pkgname can be substring
+			if [ "$provided_pkgname" = "$pkgname" ]; then
+				providers[$provider_pkgname]=$provider_pkgname
+			fi
+		done < <(xbps-query $repositories -p provides -R -s "$pkgname")
+		# leading space ensures ${[]} != ''
+		providers_cache[$pkgname]=" ${providers[*]}"
+	fi
+	echo ${providers_cache[$pkgname]}
+}
+
+conflicts_of() {
+	# print list of packages that are _marked_ as confliting with given one
+	local pkgname=$1
+	local in_conflict provider
+	if [ "${conflicts_cache[$pkgname]}" = '' ]; then
+		local -A all
+		while read -r in_conflict; do
+			in_conflict=${in_conflict%'<'*}
+			in_conflict=${in_conflict%'>'*}
+			providers_of "$in_conflict" > /dev/null # executing in same process to fill cache
+			for provider in $(providers_of "$in_conflict"); do
+				all[$provider]=$provider
+			done
+		done < <(xbps-query $repositories -p conflicts,replaces -R "$pkgname")
+		# leading space ensures ${[]} != ''
+		conflicts_cache[$pkgname]=" ${all[*]}"
+	fi
+	echo ${conflicts_cache[$pkgname]}
+}
+
+conflict_between() {
+	# exit successfully if packages are _marked_ as conflicting
+	if [ "$1" = "$2" ]; then
+		return 0 # can't install twice ;)
+	fi
+	conflicts_of "$1" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$1") " in
+		*" $2 "*) return 0
+	esac
+	conflicts_of "$2" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$2") " in
+		*" $1 "*) return 0
+	esac
+	return 1
+}
+
+list_newly_built_files() {
+	# print one line per file in newly build packages
+	# each line contains pkgname and file path
+	local pkg subpkg file
+	while read -r pkg; do
+		for subpkg in $(xsubpkg "$pkg"); do
+			while read -r file; do
+				echo "$subpkg" "$file"
+			done < <(xbps-query $repositories -i -f "$subpkg" | sed s'/ -> .*//')
+		done
+	done < /tmp/templates
+}
+
+list_interesting_files() {
+	# list files potentially contained in more than one package
+	# each line contains pkgname and file path
+	if partial_check; then
+		list_newly_built_files
+	else
+		xlocate / | sed s'/ -> .*//' | grep -F -f <(xlocate / | cut -f 2- | sed s'/ -> .*//' | sort | uniq -d)
+	fi
+}
+
+group_by_file_full() {
+	# create associative array `owners` mapping file to list of packages
+	# for packages potentially conflicting with newly built ones
+	local pkgver file pkgname
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(list_interesting_files)
+}
+
+group_by_file_partial() {
+	# create associative array `owners` mapping file to list of packages
+	# for all packages in xlocate
+	local pkgname file pkgver
+	## newly build packages
+	while read -r pkgname file; do
+		owners[$file]+=" $pkgname"
+		newly_built[$pkgname]=$pkgname
+	done < <(list_newly_built_files)
+	## rest of repository
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if [ -z "${newly_built[$pkgname]}" ] && template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(xlocate / | sed s'/ -> .*//' | grep -F -f <(list_newly_built_files | cut -d ' ' -f 2-))
+}
+
+group_by_pair() {
+	# find package pairs owning same file and not marked as conflicting
+	local pkg file a b
+	while read -r pkg file; do
+		for a in ${owners[$file]}; do
+			for b in ${owners[$file]}; do
+				if ! [ "$a" "<" "$b" ]; then
+					continue
+				fi
+				if partial_check && [ -z "${newly_built[$a]}" ] && [ -z "${newly_built[$b]}" ]; then
+					continue
+				fi
+				if ! conflict_between "$a" "$b"; then
+					unset pair_files
+					local -A pair_files
+					eval "${pairs["$a $b"]}"
+					pair_files[$file]="$file"
+					pairs["$a $b"]="${pair_files[@]@A}"
+				fi
+			done
+		done
+	done < <(list_interesting_files)
+}
+
+print_out() {
+	local pair file
+	if [ 0 = "${#pairs[@]}" ]; then
+		echo 1>&2 "No conflicts found"
+		exit 0
+	fi
+	while read -r pair; do
+		rv=1
+		echo "${pair% *} and ${pair#* } conflict for"
+		unset pair_files
+		eval "${pairs[$pair]}"
+		for file in "${pair_files[@]}"; do
+			echo "  $file"
+		done | sort
+	done < <(printf '%s\n' "${!pairs[@]}" | sort)
+}
+
+if partial_check; then
+	group_by_file_partial
+else
+	group_by_file_full
+fi
+group_by_pair
+print_out
+
+exit $rv

From a93c62ce18158fc1259e171b6f54dd59c18d58d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Tue, 26 Apr 2022 17:13:30 +0200
Subject: [PATCH 3/3] .github: check file conflicts in x86_64 build job

---
 .github/workflows/build.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 2ae7a3e728fa..f72d6f6c2c43 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -94,6 +94,12 @@ jobs:
           "$here/common/travis/show_files.sh" "$BOOTSTRAP" "$ARCH"
           )
 
+      - name: Check file conflicts
+        if: matrix.config.arch == 'x86_64' # the arch indexed in xlocate
+        run: |
+          xlocate -S &&
+          common/scripts/lint-conflicts
+
       - name: Verify repository state
         run: |
           (

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[classabbyamp]

[-- Attachment #1: Type: text/plain, Size: 191 bytes --]

New review comment by classabbyamp on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r862566012

Comment:
nit/typo: `confliting` -> `conflicting`

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[classabbyamp]

[-- Attachment #1: Type: text/plain, Size: 180 bytes --]

New review comment by classabbyamp on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r862566708

Comment:
nit/typo: `build` -> `built`

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[classabbyamp]

[-- Attachment #1: Type: text/plain, Size: 173 bytes --]

New review comment by classabbyamp on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r862567655

Comment:
same nit/typo (built)

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[classabbyamp]

[-- Attachment #1: Type: text/plain, Size: 189 bytes --]

New review comment by classabbyamp on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r862568072

Comment:
this can be switched back now, right?

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[classabbyamp]

[-- Attachment #1: Type: text/plain, Size: 196 bytes --]

New review comment by classabbyamp on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r862565817

Comment:
shouldn't this be `-f` (exists and is file)?

Re: [PR PATCH] [Updated] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 424 bytes --]

There is an updated pull request by Chocimier against master on the void-packages repository

https://github.com/Chocimier/void-packages-org lint-conflicts
https://github.com/void-linux/void-packages/pull/36876

Check file conflicts in x86_64 build job
Report packages installing same file and not marked with conflicts or replaces.

A patch file from https://github.com/void-linux/void-packages/pull/36876.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-lint-conflicts-36876.patch --]
[-- Type: text/x-diff, Size: 7201 bytes --]

From e930d3bc7094feb596b98b2ad2d3f582ecf03555 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Wed, 4 May 2022 21:44:51 +0200
Subject: [PATCH 1/2] common/: add script finding file conflicts

---
 common/scripts/lint-conflicts | 195 ++++++++++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)
 create mode 100755 common/scripts/lint-conflicts

diff --git a/common/scripts/lint-conflicts b/common/scripts/lint-conflicts
new file mode 100755
index 000000000000..10fe7a20b6f9
--- /dev/null
+++ b/common/scripts/lint-conflicts
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+# Report packages installing same file and not marked with
+# conflicts or replaces.
+# Without argument, find conflicts between packages in local
+# repository at hostdir/binpkgs and packages indexed in xlocate.
+# With single path as argument, read that local repository.
+# With -a flag, find conflicts between packages indexed in xlocate.
+
+if [ "$#" = 0 ]; then
+	binpkgs="$PWD/hostdir/binpkgs"
+elif [ "$1" = -a ]; then
+	all=1
+elif [ -d "$1" ]; then
+	binpkgs="$1"
+else
+	echo "Usage:"
+	echo "$0"
+	echo "  check packages in ./hostdir/binpkgs"
+	echo "$0 path/to/hostdir/binpkgs"
+	echo "  check packages there"
+	echo "$0 -a"
+	echo "  check all packages indexed in xlocate"
+	exit 1
+fi
+
+declare -A newly_built conflicts_cache providers_cache pairs owners
+repositories=("--repository=${binpkgs}" "--repository=${binpkgs}/nonfree")
+rv=0
+
+template_exists() {
+	[ -f "srcpkgs/$1/template" ]
+}
+
+partial_check() {
+	[ -z "$all" ]
+}
+
+providers_of() {
+	# print the pkgname and packages that `provides` it
+	local pkgname=$1
+	if [ "${providers_cache[$pkgname]}" = '' ]; then
+		local line provider_pkgver provided_pkgver provider_pkgname provided_pkgname
+		local -A providers
+		providers[$pkgname]=$pkgname
+		while read -r line; do
+			line=${line%%'('*}
+			provider_pkgver=${line%': '*}
+			provided_pkgver=${line#*': '}
+			provider_pkgname=${provider_pkgver%-*}
+			provided_pkgname=${provided_pkgver%-*}
+			# comes from $(xbps-query -s $pkgname), so $pkgname can be substring
+			if [ "$provided_pkgname" = "$pkgname" ]; then
+				providers[$provider_pkgname]=$provider_pkgname
+			fi
+		done < <(xbps-query "${repositories[@]}" -p provides -R -s "$pkgname")
+		# leading space ensures ${[]} != ''
+		providers_cache[$pkgname]=" ${providers[*]}"
+	fi
+	echo ${providers_cache[$pkgname]}
+}
+
+conflicts_of() {
+	# print list of packages that are _marked_ as confliting with given one
+	local pkgname=$1
+	if [ "${conflicts_cache[$pkgname]}" = '' ]; then
+		local in_conflict provider
+		local -A all
+		while read -r in_conflict; do
+			in_conflict=${in_conflict%'<'*}
+			in_conflict=${in_conflict%'>'*}
+			providers_of "$in_conflict" > /dev/null # executing in same process to fill cache
+			for provider in $(providers_of "$in_conflict"); do
+				all[$provider]=$provider
+			done
+		done < <(xbps-query "${repositories[@]}" -p conflicts,replaces -R "$pkgname")
+		# leading space ensures ${[]} != ''
+		conflicts_cache[$pkgname]=" ${all[*]}"
+	fi
+	echo ${conflicts_cache[$pkgname]}
+}
+
+conflict_between() {
+	# exit successfully if packages are _marked_ as conflicting
+	conflicts_of "$1" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$1") " in
+		*" $2 "*) return 0
+	esac
+	conflicts_of "$2" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$2") " in
+		*" $1 "*) return 0
+	esac
+	return 1
+}
+
+list_newly_built_files() {
+	# print one line per file in newly build packages
+	# each line contains pkgname and file path
+	local pkgver pkgname
+	while read -r pkgver; do
+		pkgname=${pkgver%-*}
+		xbps-query "${repositories[@]}" -i -f "$pkgname" | sed s'/ -> .*//;'" s/^/$pkgname /"
+	done < <(xbps-query "${repositories[@]}" -i -R -s '' | cut -d' ' -f 2)
+}
+
+list_interesting_files() {
+	# list files potentially contained in more than one package
+	# each line contains pkgver/pkgname and file path
+	if partial_check; then
+		list_newly_built_files
+	else
+		xlocate / | sed s'/ -> .*//' | grep -F -f <(xlocate / | cut -f 2- | sed s'/ -> .*//' | sort | uniq -d)
+	fi
+}
+
+group_by_file_full() {
+	# create associative array `owners` mapping file to list of packages
+	# for packages potentially conflicting with newly built ones
+	local pkgver file pkgname
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(list_interesting_files)
+}
+
+group_by_file_partial() {
+	# create associative array `owners` mapping file to list of packages
+	# for all packages in xlocate
+	local pkgname file
+	## newly build packages
+	while read -r pkgname file; do
+		owners[$file]+=" $pkgname"
+		newly_built[$pkgname]=$pkgname
+	done < <(list_newly_built_files)
+	## rest of repository
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if [ -z "${newly_built[$pkgname]}" ] && template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(xlocate / | sed s'/ -> .*//' | grep -F -f <(list_newly_built_files | cut -d ' ' -f 2-))
+}
+
+group_by_pair() {
+	# find package pairs owning same file and not marked as conflicting
+	local pkg file a b
+	while read -r pkg file; do
+		for a in ${owners[$file]}; do
+			for b in ${owners[$file]}; do
+				if ! [ "$a" "<" "$b" ]; then
+					continue
+				fi
+				if partial_check && [ -z "${newly_built[$a]}" ] && [ -z "${newly_built[$b]}" ]; then
+					continue
+				fi
+				if ! conflict_between "$a" "$b"; then
+					unset pair_files
+					local -A pair_files
+					eval "${pairs["$a $b"]}"
+					pair_files[$file]="$file"
+					pairs["$a $b"]="${pair_files[@]@A}"
+				fi
+			done
+		done
+	done < <(list_interesting_files)
+}
+
+print_out() {
+	local pair file
+	if [ "${#pairs[@]}" = 0 ]; then
+		echo 1>&2 "No conflicts found in" "${repositories[@]#*=}"
+		exit 0
+	fi
+	while read -r pair; do
+		rv=1
+		echo "${pair% *} and ${pair#* } conflict for"
+		unset pair_files
+		eval "${pairs[$pair]}"
+		for file in "${pair_files[@]}"; do
+			echo "  $file"
+		done | sort
+	done < <(printf '%s\n' "${!pairs[@]}" | sort)
+}
+
+if partial_check; then
+	group_by_file_partial
+else
+	group_by_file_full
+fi
+group_by_pair
+print_out
+
+exit $rv

From 842f7048fc51cf09ea1ddcd00949ab0851eff1c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Wed, 4 May 2022 21:44:51 +0200
Subject: [PATCH 2/2] .github: check file conflicts in x86_64 build job

---
 .github/workflows/build.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 2ae7a3e728fa..e4619a3f3c4b 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -94,6 +94,12 @@ jobs:
           "$here/common/travis/show_files.sh" "$BOOTSTRAP" "$ARCH"
           )
 
+      - name: Check file conflicts
+        if: matrix.config.arch == 'x86_64' # the arch indexed in xlocate
+        run: |
+          xlocate -S &&
+          common/scripts/lint-conflicts $HOME/hostdir/binpkgs
+
       - name: Verify repository state
         run: |
           (

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 161 bytes --]

New review comment by Chocimier on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r865383885

Comment:
Right, done.

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 156 bytes --]

New review comment by Chocimier on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r865384084

Comment:
changed

Re: Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 518 bytes --]

New comment by Chocimier on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#issuecomment-1117940370

Comment:
Changed usage so it takes list of packages from local repository index and not global file. Can be easily used locally without arguments.

Test runs at <https://github.com/Chocimier/void-packages-org/actions/runs/2272244355>, <https://github.com/Chocimier/void-packages-org/actions/runs/2272208791>, <https://github.com/Chocimier/void-packages-org/actions/runs/2272194034>

Re: [PR PATCH] [Updated] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 424 bytes --]

There is an updated pull request by Chocimier against master on the void-packages repository

https://github.com/Chocimier/void-packages-org lint-conflicts
https://github.com/void-linux/void-packages/pull/36876

Check file conflicts in x86_64 build job
Report packages installing same file and not marked with conflicts or replaces.

A patch file from https://github.com/void-linux/void-packages/pull/36876.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-lint-conflicts-36876.patch --]
[-- Type: text/x-diff, Size: 7201 bytes --]

From cad98be396422d68a9c742e1c5fefba8f75d0b70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Wed, 4 May 2022 21:44:51 +0200
Subject: [PATCH 1/2] common/: add script finding file conflicts

---
 common/scripts/lint-conflicts | 195 ++++++++++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)
 create mode 100755 common/scripts/lint-conflicts

diff --git a/common/scripts/lint-conflicts b/common/scripts/lint-conflicts
new file mode 100755
index 000000000000..d31cbe5b54ad
--- /dev/null
+++ b/common/scripts/lint-conflicts
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+# Report packages installing same file and not marked with
+# conflicts or replaces.
+# Without argument, find conflicts between packages in local
+# repository at hostdir/binpkgs and packages indexed in xlocate.
+# With single path as argument, read that local repository.
+# With -a flag, find conflicts between packages indexed in xlocate.
+
+if [ "$#" = 0 ]; then
+	binpkgs="$PWD/hostdir/binpkgs"
+elif [ "$1" = -a ]; then
+	all=1
+elif [ -d "$1" ]; then
+	binpkgs="$1"
+else
+	echo "Usage:"
+	echo "$0"
+	echo "  check packages in ./hostdir/binpkgs"
+	echo "$0 path/to/hostdir/binpkgs"
+	echo "  check packages there"
+	echo "$0 -a"
+	echo "  check all packages indexed in xlocate"
+	exit 1
+fi
+
+declare -A newly_built conflicts_cache providers_cache pairs owners
+repositories=("--repository=${binpkgs}" "--repository=${binpkgs}/nonfree")
+rv=0
+
+template_exists() {
+	[ -f "srcpkgs/$1/template" ]
+}
+
+partial_check() {
+	[ -z "$all" ]
+}
+
+providers_of() {
+	# print the pkgname and packages that `provides` it
+	local pkgname=$1
+	if [ "${providers_cache[$pkgname]}" = '' ]; then
+		local line provider_pkgver provided_pkgver provider_pkgname provided_pkgname
+		local -A providers
+		providers[$pkgname]=$pkgname
+		while read -r line; do
+			line=${line%%'('*}
+			provider_pkgver=${line%': '*}
+			provided_pkgver=${line#*': '}
+			provider_pkgname=${provider_pkgver%-*}
+			provided_pkgname=${provided_pkgver%-*}
+			# comes from $(xbps-query -s $pkgname), so $pkgname can be substring
+			if [ "$provided_pkgname" = "$pkgname" ]; then
+				providers[$provider_pkgname]=$provider_pkgname
+			fi
+		done < <(xbps-query "${repositories[@]}" -p provides -R -s "$pkgname")
+		# leading space ensures ${[]} != ''
+		providers_cache[$pkgname]=" ${providers[*]}"
+	fi
+	echo ${providers_cache[$pkgname]}
+}
+
+conflicts_of() {
+	# print list of packages that are _marked_ as confliting with given one
+	local pkgname=$1
+	if [ "${conflicts_cache[$pkgname]}" = '' ]; then
+		local in_conflict provider
+		local -A all
+		while read -r in_conflict; do
+			in_conflict=${in_conflict%'<'*}
+			in_conflict=${in_conflict%'>'*}
+			providers_of "$in_conflict" > /dev/null # executing in same process to fill cache
+			for provider in $(providers_of "$in_conflict"); do
+				all[$provider]=$provider
+			done
+		done < <(xbps-query "${repositories[@]}" -p conflicts,replaces -R "$pkgname")
+		# leading space ensures ${[]} != ''
+		conflicts_cache[$pkgname]=" ${all[*]}"
+	fi
+	echo ${conflicts_cache[$pkgname]}
+}
+
+conflict_between() {
+	# exit successfully if packages are _marked_ as conflicting
+	conflicts_of "$1" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$1") " in
+		*" $2 "*) return 0
+	esac
+	conflicts_of "$2" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$2") " in
+		*" $1 "*) return 0
+	esac
+	return 1
+}
+
+list_newly_built_files() {
+	# print one line per file in newly built packages
+	# each line contains pkgname and file path
+	local pkgver pkgname
+	while read -r pkgver; do
+		pkgname=${pkgver%-*}
+		xbps-query "${repositories[@]}" -i -f "$pkgname" | sed s'/ -> .*//;'" s/^/$pkgname /"
+	done < <(xbps-query "${repositories[@]}" -i -R -s '' | cut -d' ' -f 2)
+}
+
+list_interesting_files() {
+	# list files potentially contained in more than one package
+	# each line contains pkgver/pkgname and file path
+	if partial_check; then
+		list_newly_built_files
+	else
+		xlocate / | sed s'/ -> .*//' | grep -F -f <(xlocate / | cut -f 2- | sed s'/ -> .*//' | sort | uniq -d)
+	fi
+}
+
+group_by_file_full() {
+	# create associative array `owners` mapping file to list of packages
+	# for packages potentially conflicting with newly built ones
+	local pkgver file pkgname
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(list_interesting_files)
+}
+
+group_by_file_partial() {
+	# create associative array `owners` mapping file to list of packages
+	# for all packages in xlocate
+	local pkgname file
+	## newly built packages
+	while read -r pkgname file; do
+		owners[$file]+=" $pkgname"
+		newly_built[$pkgname]=$pkgname
+	done < <(list_newly_built_files)
+	## rest of repository
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if [ -z "${newly_built[$pkgname]}" ] && template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(xlocate / | sed s'/ -> .*//' | grep -F -f <(list_newly_built_files | cut -d ' ' -f 2-))
+}
+
+group_by_pair() {
+	# find package pairs owning same file and not marked as conflicting
+	local pkg file a b
+	while read -r pkg file; do
+		for a in ${owners[$file]}; do
+			for b in ${owners[$file]}; do
+				if ! [ "$a" "<" "$b" ]; then
+					continue
+				fi
+				if partial_check && [ -z "${newly_built[$a]}" ] && [ -z "${newly_built[$b]}" ]; then
+					continue
+				fi
+				if ! conflict_between "$a" "$b"; then
+					unset pair_files
+					local -A pair_files
+					eval "${pairs["$a $b"]}"
+					pair_files[$file]="$file"
+					pairs["$a $b"]="${pair_files[@]@A}"
+				fi
+			done
+		done
+	done < <(list_interesting_files)
+}
+
+print_out() {
+	local pair file
+	if [ "${#pairs[@]}" = 0 ]; then
+		echo 1>&2 "No conflicts found in" "${repositories[@]#*=}"
+		exit 0
+	fi
+	while read -r pair; do
+		rv=1
+		echo "${pair% *} and ${pair#* } conflict for"
+		unset pair_files
+		eval "${pairs[$pair]}"
+		for file in "${pair_files[@]}"; do
+			echo "  $file"
+		done | sort
+	done < <(printf '%s\n' "${!pairs[@]}" | sort)
+}
+
+if partial_check; then
+	group_by_file_partial
+else
+	group_by_file_full
+fi
+group_by_pair
+print_out
+
+exit $rv

From fe2c827a526c94235189fb57d5c403b6da1346a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Wed, 4 May 2022 21:44:51 +0200
Subject: [PATCH 2/2] .github: check file conflicts in x86_64 build job

---
 .github/workflows/build.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 2ae7a3e728fa..e4619a3f3c4b 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -94,6 +94,12 @@ jobs:
           "$here/common/travis/show_files.sh" "$BOOTSTRAP" "$ARCH"
           )
 
+      - name: Check file conflicts
+        if: matrix.config.arch == 'x86_64' # the arch indexed in xlocate
+        run: |
+          xlocate -S &&
+          common/scripts/lint-conflicts $HOME/hostdir/binpkgs
+
       - name: Verify repository state
         run: |
           (

Re: [PR PATCH] [Updated] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 424 bytes --]

There is an updated pull request by Chocimier against master on the void-packages repository

https://github.com/Chocimier/void-packages-org lint-conflicts
https://github.com/void-linux/void-packages/pull/36876

Check file conflicts in x86_64 build job
Report packages installing same file and not marked with conflicts or replaces.

A patch file from https://github.com/void-linux/void-packages/pull/36876.patch is attached

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-lint-conflicts-36876.patch --]
[-- Type: text/x-diff, Size: 7202 bytes --]

From 1f0fcc138a1a04c9c9de0fc861eee069aa40acd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Wed, 4 May 2022 21:44:51 +0200
Subject: [PATCH 1/2] common/: add script finding file conflicts

---
 common/scripts/lint-conflicts | 195 ++++++++++++++++++++++++++++++++++
 1 file changed, 195 insertions(+)
 create mode 100755 common/scripts/lint-conflicts

diff --git a/common/scripts/lint-conflicts b/common/scripts/lint-conflicts
new file mode 100755
index 000000000000..bb0e2b2fd6b3
--- /dev/null
+++ b/common/scripts/lint-conflicts
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+# Report packages installing same file and not marked with
+# conflicts or replaces.
+# Without argument, find conflicts between packages in local
+# repository at hostdir/binpkgs and packages indexed in xlocate.
+# With single path as argument, read that local repository.
+# With -a flag, find conflicts between packages indexed in xlocate.
+
+if [ "$#" = 0 ]; then
+	binpkgs="$PWD/hostdir/binpkgs"
+elif [ "$1" = -a ]; then
+	all=1
+elif [ -d "$1" ]; then
+	binpkgs="$1"
+else
+	echo "Usage:"
+	echo "$0"
+	echo "  check packages in ./hostdir/binpkgs"
+	echo "$0 path/to/hostdir/binpkgs"
+	echo "  check packages there"
+	echo "$0 -a"
+	echo "  check all packages indexed in xlocate"
+	exit 1
+fi
+
+declare -A newly_built conflicts_cache providers_cache pairs owners
+repositories=("--repository=${binpkgs}" "--repository=${binpkgs}/nonfree")
+rv=0
+
+template_exists() {
+	[ -f "srcpkgs/$1/template" ]
+}
+
+partial_check() {
+	[ -z "$all" ]
+}
+
+providers_of() {
+	# print the pkgname and packages that `provides` it
+	local pkgname=$1
+	if [ "${providers_cache[$pkgname]}" = '' ]; then
+		local line provider_pkgver provided_pkgver provider_pkgname provided_pkgname
+		local -A providers
+		providers[$pkgname]=$pkgname
+		while read -r line; do
+			line=${line%%'('*}
+			provider_pkgver=${line%': '*}
+			provided_pkgver=${line#*': '}
+			provider_pkgname=${provider_pkgver%-*}
+			provided_pkgname=${provided_pkgver%-*}
+			# comes from $(xbps-query -s $pkgname), so $pkgname can be substring
+			if [ "$provided_pkgname" = "$pkgname" ]; then
+				providers[$provider_pkgname]=$provider_pkgname
+			fi
+		done < <(xbps-query "${repositories[@]}" -p provides -R -s "$pkgname")
+		# leading space ensures ${[]} != ''
+		providers_cache[$pkgname]=" ${providers[*]}"
+	fi
+	echo ${providers_cache[$pkgname]}
+}
+
+conflicts_of() {
+	# print list of packages that are _marked_ as conflicting with given one
+	local pkgname=$1
+	if [ "${conflicts_cache[$pkgname]}" = '' ]; then
+		local in_conflict provider
+		local -A all
+		while read -r in_conflict; do
+			in_conflict=${in_conflict%'<'*}
+			in_conflict=${in_conflict%'>'*}
+			providers_of "$in_conflict" > /dev/null # executing in same process to fill cache
+			for provider in $(providers_of "$in_conflict"); do
+				all[$provider]=$provider
+			done
+		done < <(xbps-query "${repositories[@]}" -p conflicts,replaces -R "$pkgname")
+		# leading space ensures ${[]} != ''
+		conflicts_cache[$pkgname]=" ${all[*]}"
+	fi
+	echo ${conflicts_cache[$pkgname]}
+}
+
+conflict_between() {
+	# exit successfully if packages are _marked_ as conflicting
+	conflicts_of "$1" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$1") " in
+		*" $2 "*) return 0
+	esac
+	conflicts_of "$2" > /dev/null # executing in same process to fill cache
+	case " $(conflicts_of "$2") " in
+		*" $1 "*) return 0
+	esac
+	return 1
+}
+
+list_newly_built_files() {
+	# print one line per file in newly built packages
+	# each line contains pkgname and file path
+	local pkgver pkgname
+	while read -r pkgver; do
+		pkgname=${pkgver%-*}
+		xbps-query "${repositories[@]}" -i -f "$pkgname" | sed s'/ -> .*//;'" s/^/$pkgname /"
+	done < <(xbps-query "${repositories[@]}" -i -R -s '' | cut -d' ' -f 2)
+}
+
+list_interesting_files() {
+	# list files potentially contained in more than one package
+	# each line contains pkgver/pkgname and file path
+	if partial_check; then
+		list_newly_built_files
+	else
+		xlocate / | sed s'/ -> .*//' | grep -F -f <(xlocate / | cut -f 2- | sed s'/ -> .*//' | sort | uniq -d)
+	fi
+}
+
+group_by_file_full() {
+	# create associative array `owners` mapping file to list of packages
+	# for packages potentially conflicting with newly built ones
+	local pkgver file pkgname
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(list_interesting_files)
+}
+
+group_by_file_partial() {
+	# create associative array `owners` mapping file to list of packages
+	# for all packages in xlocate
+	local pkgname file
+	## newly built packages
+	while read -r pkgname file; do
+		owners[$file]+=" $pkgname"
+		newly_built[$pkgname]=$pkgname
+	done < <(list_newly_built_files)
+	## rest of repository
+	while read -r pkgver file; do
+		pkgname=${pkgver%-*}
+		if [ -z "${newly_built[$pkgname]}" ] && template_exists "$pkgname"; then
+			owners[$file]+=" $pkgname"
+		fi
+	done < <(xlocate / | sed s'/ -> .*//' | grep -F -f <(list_newly_built_files | cut -d ' ' -f 2-))
+}
+
+group_by_pair() {
+	# find package pairs owning same file and not marked as conflicting
+	local pkg file a b
+	while read -r pkg file; do
+		for a in ${owners[$file]}; do
+			for b in ${owners[$file]}; do
+				if ! [ "$a" "<" "$b" ]; then
+					continue
+				fi
+				if partial_check && [ -z "${newly_built[$a]}" ] && [ -z "${newly_built[$b]}" ]; then
+					continue
+				fi
+				if ! conflict_between "$a" "$b"; then
+					unset pair_files
+					local -A pair_files
+					eval "${pairs["$a $b"]}"
+					pair_files[$file]="$file"
+					pairs["$a $b"]="${pair_files[@]@A}"
+				fi
+			done
+		done
+	done < <(list_interesting_files)
+}
+
+print_out() {
+	local pair file
+	if [ "${#pairs[@]}" = 0 ]; then
+		echo 1>&2 "No conflicts found in" "${repositories[@]#*=}"
+		exit 0
+	fi
+	while read -r pair; do
+		rv=1
+		echo "${pair% *} and ${pair#* } conflict for"
+		unset pair_files
+		eval "${pairs[$pair]}"
+		for file in "${pair_files[@]}"; do
+			echo "  $file"
+		done | sort
+	done < <(printf '%s\n' "${!pairs[@]}" | sort)
+}
+
+if partial_check; then
+	group_by_file_partial
+else
+	group_by_file_full
+fi
+group_by_pair
+print_out
+
+exit $rv

From 01301eb55117e3a5cc0dfb58b986bd33f2979686 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <chocimier@tlen.pl>
Date: Wed, 4 May 2022 21:44:51 +0200
Subject: [PATCH 2/2] .github: check file conflicts in x86_64 build job

---
 .github/workflows/build.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 2ae7a3e728fa..e4619a3f3c4b 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -94,6 +94,12 @@ jobs:
           "$here/common/travis/show_files.sh" "$BOOTSTRAP" "$ARCH"
           )
 
+      - name: Check file conflicts
+        if: matrix.config.arch == 'x86_64' # the arch indexed in xlocate
+        run: |
+          xlocate -S &&
+          common/scripts/lint-conflicts $HOME/hostdir/binpkgs
+
       - name: Verify repository state
         run: |
           (

Re: [PR REVIEW] Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 160 bytes --]

New review comment by Chocimier on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#discussion_r865390651

Comment:
fixed typos

Re: Check file conflicts in x86_64 build job

[classabbyamp]

[-- Attachment #1: Type: text/plain, Size: 250 bytes --]

New comment by classabbyamp on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#issuecomment-1118036791

Comment:
those test runs look good to me, maybe annotations could be added à la #36950 (if they make sense to)?

Re: Check file conflicts in x86_64 build job

[0x5c]

[-- Attachment #1: Type: text/plain, Size: 310 bytes --]

New comment by 0x5c on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#issuecomment-1118091204

Comment:
> those test runs look good to me, maybe annotations could be added à la #36950 (if they make sense to)?

As an annotation without the `file` and `line` parameters yeah

Re: Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 243 bytes --]

New comment by Chocimier on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#issuecomment-1118784376

Comment:
Without `file` message is not turned into comment elsewhere than logs, I do not see advantage then.

Re: Check file conflicts in x86_64 build job

[0x5c]

[-- Attachment #1: Type: text/plain, Size: 472 bytes --]

New comment by 0x5c on void-packages repository

https://github.com/void-linux/void-packages/pull/36876#issuecomment-1118845819

Comment:
> Without `file` message is not turned into comment elsewhere than logs, I do not see advantage then.

It gets a special mention in the action run summary page, and arguably the special highlighting in the logs is worth it. The message in the logs could otherwise be printed in red with ANSI sequences since GH actions render those

Re: [PR PATCH] [Merged]: Check file conflicts in x86_64 build job

[Chocimier]

[-- Attachment #1: Type: text/plain, Size: 253 bytes --]

There's a merged pull request on the void-packages repository

Check file conflicts in x86_64 build job
https://github.com/void-linux/void-packages/pull/36876

Description:
Report packages installing same file and not marked with conflicts or replaces.