* [PR PATCH] xbps-src: clean up fetch hook and allow mirroring
@ 2022-08-06 2:57 Duncaen
2022-08-07 17:52 ` [Draft] " CameronNemo
` (4 more replies)
0 siblings, 5 replies; 6+ messages in thread
From: Duncaen @ 2022-08-06 2:57 UTC (permalink / raw)
To: ml
[-- Attachment #1: Type: text/plain, Size: 530 bytes --]
There is a new pull request by Duncaen against master on the void-packages repository
https://github.com/Duncaen/void-packages distfiles-mirror
https://github.com/void-linux/void-packages/pull/38494
xbps-src: clean up fetch hook and allow mirroring
As an alternative to https://github.com/void-linux/void-packages/pull/38438
Allow mirroring by having multiple distfiles for the same target file name, either by url or by ">filename".
A patch file from https://github.com/void-linux/void-packages/pull/38494.patch is attached
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: github-pr-distfiles-mirror-38494.patch --]
[-- Type: text/x-diff, Size: 11020 bytes --]
From 463356e4cd1f76d6396c9f6bfdb9f150155d7fad Mon Sep 17 00:00:00 2001
From: Duncaen <duncaen@voidlinux.org>
Date: Sat, 6 Aug 2022 04:54:05 +0200
Subject: [PATCH] xbps-src: clean up fetch hook and allow mirroring
---
common/environment/build-style/R-cran.sh | 3 +-
common/hooks/do-fetch/00-distfiles.sh | 201 ++++++++++++-----------
srcpkgs/R-cran-pillar/template | 3 +-
3 files changed, 110 insertions(+), 97 deletions(-)
diff --git a/common/environment/build-style/R-cran.sh b/common/environment/build-style/R-cran.sh
index ded66793396b..905c3b9dc0c7 100644
--- a/common/environment/build-style/R-cran.sh
+++ b/common/environment/build-style/R-cran.sh
@@ -4,5 +4,6 @@ wrksrc="${XBPS_BUILDDIR}/${pkgname#R-cran-}"
# default to cran
if [ -z "$distfiles" ]; then
- distfiles="https://cran.r-project.org/src/contrib/${pkgname#R-cran-}_${version//r/-}.tar.gz"
+ distfiles="https://cran.r-project.org/src/contrib/${pkgname#R-cran-}_${version//r/-}.tar.gz
+ https://cran.r-project.org/src/contrib/Archive/${pkgname#R-cran-}/${pkgname#R-cran-}_${version//r/-}.tar.gz"
fi
diff --git a/common/hooks/do-fetch/00-distfiles.sh b/common/hooks/do-fetch/00-distfiles.sh
index e2bf54378fe0..5fa394c9c790 100644
--- a/common/hooks/do-fetch/00-distfiles.sh
+++ b/common/hooks/do-fetch/00-distfiles.sh
@@ -2,24 +2,6 @@
# the $distfiles variable and then verifies its sha256 checksum comparing
# its value with the one stored in the $checksum variable.
-# Get the checksum for $curfile at index $dfcount
-get_cksum() {
- local curfile="$1" dfcount="$2" ckcount cksum i
-
- ckcount=0
- cksum=0
- for i in ${checksum}; do
- if [ $dfcount -eq $ckcount -a -n "$i" ]; then
- cksum=$i
- fi
- ckcount=$((ckcount + 1))
- done
- if [ -z "$cksum" ]; then
- msg_error "$pkgver: cannot find checksum for $curfile.\n"
- fi
- echo "$cksum"
-}
-
# Return the checksum of the contents of a tarball
contents_cksum() {
local curfile="$1" cursufx cksum
@@ -110,9 +92,7 @@ contents_cksum() {
# Verify the checksum for $curfile stored at $distfile and index $dfcount
verify_cksum() {
- local curfile="$1" distfile="$2" dfcount="$3" filesum cksum
-
- cksum=$(get_cksum $curfile $dfcount)
+ local curfile="$1" distfile="$2" cksum="$3" filesum
# If the checksum starts with an commercial at (@) it is the contents checksum
if [ "${cksum:0:1}" = "@" ]; then
@@ -121,7 +101,7 @@ verify_cksum() {
filesum=$(contents_cksum "$curfile")
if [ "${cksum}" != "$filesum" ]; then
echo
- msg_red "SHA256 mismatch for '$curfile:'\n@$filesum\n"
+ msg_red "SHA256 mismatch for '${curfile}:'\n@${filesum}\n"
errors=$((errors + 1))
else
msg_normal_append "OK.\n"
@@ -131,7 +111,7 @@ verify_cksum() {
filesum=$(${XBPS_DIGEST_CMD} "$distfile")
if [ "$cksum" != "$filesum" ]; then
echo
- msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n"
+ msg_red "SHA256 mismatch for '${curfile}:'\n${filesum}\n"
errors=$((errors + 1))
else
if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then
@@ -145,22 +125,20 @@ verify_cksum() {
# Link an existing cksum $distfile for $curfile at index $dfcount
link_cksum() {
- local curfile="$1" distfile="$2" dfcount="$3" filesum cksum
-
- cksum=$(get_cksum $curfile $dfcount)
-
+ local curfile="$1" distfile="$2" cksum="$3"
if [ -n "$cksum" -a -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then
ln -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" "$distfile"
msg_normal "$pkgver: using known distfile $curfile.\n"
+ return 0
fi
+ return 1
}
try_mirrors() {
- local curfile="$1" distfile="$2" dfcount="$3" subdir="$4" f="$5"
- local filesum cksum basefile mirror path scheme
- [ -z "$XBPS_DISTFILES_MIRROR" ] && return
+ local curfile="$1" distfile="$2" cksum="$3" f="$4"
+ local filesum basefile mirror path scheme good
+ [ -z "$XBPS_DISTFILES_MIRROR" ] && return 1
basefile="${f##*/}"
- cksum=$(get_cksum $curfile $dfcount)
for mirror in $XBPS_DISTFILES_MIRROR; do
scheme="file"
if [[ $mirror == *://* ]]; then
@@ -179,28 +157,80 @@ try_mirrors() {
fi
if [[ "$mirror" == *voidlinux* ]]; then
# For distfiles.voidlinux.* append the subdirectory
- mirror="$mirror/$subdir"
+ mirror="$mirror/$pkgname-$version"
fi
- msg_normal "$pkgver: fetching distfile '$curfile' from '$mirror'...\n"
+ msg_normal "$pkgver: fetching distfile '$curfile' from mirror '$mirror'...\n"
$fetch_cmd "$mirror/$curfile"
# If basefile was not found, but a curfile file may exist, try to fetch it
- if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then
- $fetch_cmd "$mirror/$basefile"
- fi
+ # if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then
+ # msg_normal "$pkgver: fetching distfile '$basefile' from mirror '$mirror'...\n"
+ # $fetch_cmd "$mirror/$basefile"
+ # fi
[ ! -f "$distfile" ] && continue
flock -n ${distfile}.part rm -f ${distfile}.part
filesum=$(${XBPS_DIGEST_CMD} "$distfile")
- [ "$cksum" == "$filesum" ] && break
+ if [ "$cksum" == "$filesum" ]; then
+ return 0
+ fi
msg_normal "$pkgver: checksum failed - removing '$curfile'...\n"
rm -f ${distfile}
done
+ return 1
+}
+
+try_urls() {
+ local curfile="$1"
+ local good=
+ for i in ${_file_idxs["$curfile"]}; do
+ local cksum=${_checksums["$i"]}
+ local url=${_distfiles["$i"]}
+
+ # If distfile does not exist, download it from the original location.
+ if [[ "$FTP_RETRIES" && "${url}" =~ ^ftp:// ]]; then
+ max_retries="$FTP_RETRIES"
+ else
+ max_retries=1
+ fi
+ for retry in $(seq 1 1 $max_retries); do
+ if [ ! -f "$distfile" ]; then
+ if [ "$retry" == 1 ]; then
+ msg_normal "$pkgver: fetching distfile '$curfile' from '$url'...\n"
+ else
+ msg_normal "$pkgver: fetch attempt $retry of $max_retries...\n"
+ fi
+ flock "${distfile}.part" $fetch_cmd "$url"
+ fi
+ done
+
+ if [ ! -f "$distfile" ]; then
+ continue
+ fi
+
+ # distfile downloaded, verify sha256 hash.
+ flock -n "${distfile}.part" rm -f "${distfile}.part"
+ verify_cksum "$f" "$distfile" "$cksum"
+ return 0
+ done
+ return 1
}
hook() {
local srcdir="$XBPS_SRCDISTDIR/$pkgname-$version"
local dfcount=0 dfgood=0 errors=0 max_retries
- if [ ! -d "$srcdir" ]; then
+ local -a _distfiles=($distfiles)
+ local -a _checksums=($checksum)
+ local -A _file_idxs
+
+ # Create a map from target file to index in _distfiles/_checksums
+ for i in ${!_distfiles[@]}; do
+ f="${_distfiles[$i]}"
+ curfile="${f#*>}"
+ curfile="${curfile##*/}"
+ _file_idxs["$curfile"]+=" $i"
+ done
+
+ if [[ ! -d "$srcdir" ]]; then
mkdir -p -m775 "$srcdir"
chgrp $(id -g) "$srcdir"
fi
@@ -212,87 +242,68 @@ hook() {
# Detect bsdtar and GNU tar (in that order of preference)
TAR_CMD="$(command -v bsdtar)"
- if [ -z "$TAR_CMD" ]; then
+ if [[ -z "$TAR_CMD" ]]; then
TAR_CMD="$(command -v tar)"
fi
# Detect distfiles with obsolete checksum and purge them from the cache
- for f in ${distfiles}; do
- curfile="${f#*>}"
- curfile="${curfile##*/}"
- distfile="$srcdir/$curfile"
-
- if [ -f "$distfile" ]; then
- cksum=$(get_cksum $curfile $dfcount)
- if [ "${cksum:0:1}" = "@" ]; then
- cksum=${cksum:1}
- filesum=$(contents_cksum "$distfile")
- else
- filesum=$(${XBPS_DIGEST_CMD} "$distfile")
- fi
- if [ "$cksum" = "$filesum" ]; then
- dfgood=$((dfgood + 1))
- else
- inode=$(stat "$distfile" --printf "%i")
- msg_warn "$pkgver: wrong checksum found for ${curfile} - purging\n"
- find ${XBPS_SRCDISTDIR} -inum ${inode} -delete -print
+ for f in ${!_file_idxs[@]}; do
+ distfile="$srcdir/$f"
+ for i in ${_file_idxs["$f"]}; do
+ if [[ -f $distfile ]]; then
+ cksum=${_checksums["$i"]}
+ if [[ ${cksum:0:1} = @ ]]; then
+ cksum=${cksum:1}
+ filesum=$(contents_cksum "$distfile")
+ else
+ filesum=$(${XBPS_DIGEST_CMD} "$distfile")
+ fi
+ if [[ $cksum = $filesum ]]; then
+ dfgood=$((dfgood + 1))
+ else
+ inode=$(stat "$distfile" --printf "%i")
+ msg_warn "$pkgver: wrong checksum found for ${curfile} - purging\n"
+ find ${XBPS_SRCDISTDIR} -inum ${inode} -delete -print
+ fi
fi
- fi
- dfcount=$((dfcount + 1))
+ dfcount=$((dfcount + 1))
+ done
done
# We're done, if all distfiles were found and had good checksums
- [ $dfcount -eq $dfgood ] && return
+ [[ $dfcount -eq $dfgood ]] && return
# Download missing distfiles and verify their checksums
- dfcount=0
- for f in ${distfiles}; do
- curfile="${f#*>}"
- curfile="${curfile##*/}"
+ for curfile in ${!_file_idxs[@]}; do
distfile="$srcdir/$curfile"
# If file lock cannot be acquired wait until it's available.
- while true; do
- flock -w 1 ${distfile}.part true
- [ $? -eq 0 ] && break
+ while ! flock -w 1 "${distfile}.part" true; do
msg_warn "$pkgver: ${curfile} is already being downloaded, waiting for 1s ...\n"
done
+
+ if [[ -f "$distfile" ]]; then
+ continue
+ fi
+
# If distfile does not exist, try to link to it.
- if [ ! -f "$distfile" ]; then
- link_cksum $curfile $distfile $dfcount
+ if link_cksum "$curfile" "$distfile" "${_checksums[0]}"; then
+ continue
fi
+
# If distfile does not exist, download it from a mirror location.
- if [ ! -f "$distfile" ]; then
- try_mirrors $curfile $distfile $dfcount $pkgname-$version $f
- fi
- # If distfile does not exist, download it from the original location.
- if [[ "$FTP_RETRIES" && "${f}" =~ ^ftp:// ]]; then
- max_retries="$FTP_RETRIES"
- else
- max_retries=1
+ if try_mirrors "$curfile" "$distfile" "${_checksums[0]}" "${_distfiles[0]}"; then
+ continue
fi
- for retry in $(seq 1 1 $max_retries); do
- if [ ! -f "$distfile" ]; then
- if [ "$retry" == 1 ]; then
- msg_normal "$pkgver: fetching distfile '$curfile'...\n"
- else
- msg_normal "$pkgver: fetch attempt $retry of $max_retries...\n"
- fi
- flock "${distfile}.part" $fetch_cmd "$f"
- fi
- done
- if [ ! -f "$distfile" ]; then
- msg_error "$pkgver: failed to fetch $curfile.\n"
+
+ if ! try_urls "$curfile"; then
+ msg_error "$pkgver: failed to fetch '$curfile'.\n"
fi
- # distfile downloaded, verify sha256 hash.
- flock -n ${distfile}.part rm -f ${distfile}.part
- verify_cksum $curfile $distfile $dfcount
- dfcount=$((dfcount + 1))
done
unset TAR_CMD
- if [ $errors -gt 0 ]; then
+ if [[ $errors -gt 0 ]]; then
msg_error "$pkgver: couldn't verify distfiles, exiting...\n"
fi
}
diff --git a/srcpkgs/R-cran-pillar/template b/srcpkgs/R-cran-pillar/template
index e041a7a7c4b9..d8d381b8e4ac 100644
--- a/srcpkgs/R-cran-pillar/template
+++ b/srcpkgs/R-cran-pillar/template
@@ -11,4 +11,5 @@ short_desc="Coloured Formatting for Columns"
maintainer="Florian Wagner <florian@wagner-flo.net>"
license="GPL-3.0-or-later"
homepage="https://github.com/r-lib/pillar/"
-checksum=0ce5d15364dab761dab5b159ec2a4586ed1635f058fa13975725a1921e43e672
+checksum="0ce5d15364dab761dab5b159ec2a4586ed1635f058fa13975725a1921e43e672
+ 0ce5d15364dab761dab5b159ec2a4586ed1635f058fa13975725a1921e43e672"
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Draft] xbps-src: clean up fetch hook and allow mirroring
2022-08-06 2:57 [PR PATCH] xbps-src: clean up fetch hook and allow mirroring Duncaen
@ 2022-08-07 17:52 ` CameronNemo
2022-08-07 18:09 ` Duncaen
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: CameronNemo @ 2022-08-07 17:52 UTC (permalink / raw)
To: ml
[-- Attachment #1: Type: text/plain, Size: 254 bytes --]
New comment by CameronNemo on void-packages repository
https://github.com/void-linux/void-packages/pull/38494#issuecomment-1207455827
Comment:
Hmm do you really think it is worth the extra complexity? How many similar cases might we have besides CRAN?
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Draft] xbps-src: clean up fetch hook and allow mirroring
2022-08-06 2:57 [PR PATCH] xbps-src: clean up fetch hook and allow mirroring Duncaen
2022-08-07 17:52 ` [Draft] " CameronNemo
@ 2022-08-07 18:09 ` Duncaen
2022-08-07 18:10 ` Duncaen
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Duncaen @ 2022-08-07 18:09 UTC (permalink / raw)
To: ml
[-- Attachment #1: Type: text/plain, Size: 679 bytes --]
New comment by Duncaen on void-packages repository
https://github.com/void-linux/void-packages/pull/38494#issuecomment-1207458649
Comment:
The extra complexity isn't that big, I expect this change to reduce the number of lines, at the moment its like +11 lines.
Its basically just creating a per distfile target list of indexes into the `distfiles` and `checksum` variables and then tries to download the first matching one.
The looping already exists for `XBPS_DISTFILES_MIRROR` and the current implementation kinda already works by indexing into distfiles+checksums but in a weird way, which is why the rewrite will be more compact/simpler, by just using indexed arrays.
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Draft] xbps-src: clean up fetch hook and allow mirroring
2022-08-06 2:57 [PR PATCH] xbps-src: clean up fetch hook and allow mirroring Duncaen
2022-08-07 17:52 ` [Draft] " CameronNemo
2022-08-07 18:09 ` Duncaen
@ 2022-08-07 18:10 ` Duncaen
2022-09-06 2:34 ` CameronNemo
2022-10-01 21:29 ` [PR PATCH] [Closed]: " Duncaen
4 siblings, 0 replies; 6+ messages in thread
From: Duncaen @ 2022-08-07 18:10 UTC (permalink / raw)
To: ml
[-- Attachment #1: Type: text/plain, Size: 750 bytes --]
New comment by Duncaen on void-packages repository
https://github.com/void-linux/void-packages/pull/38494#issuecomment-1207458649
Comment:
The extra complexity isn't that big, I expect this change to reduce the number of lines, at the moment its like +11 lines.
Its basically just creating a per distfile target list of indexes into the `distfiles` and `checksum` variables and then tries to download each distfile target by looping through the per target indexes until a download is successful.
The looping already exists for `XBPS_DISTFILES_MIRROR` and the current implementation kinda already works by indexing into distfiles+checksums but in a weird way, which is why the rewrite will be more compact/simpler, by just using indexed arrays.
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [Draft] xbps-src: clean up fetch hook and allow mirroring
2022-08-06 2:57 [PR PATCH] xbps-src: clean up fetch hook and allow mirroring Duncaen
` (2 preceding siblings ...)
2022-08-07 18:10 ` Duncaen
@ 2022-09-06 2:34 ` CameronNemo
2022-10-01 21:29 ` [PR PATCH] [Closed]: " Duncaen
4 siblings, 0 replies; 6+ messages in thread
From: CameronNemo @ 2022-09-06 2:34 UTC (permalink / raw)
To: ml
[-- Attachment #1: Type: text/plain, Size: 284 bytes --]
New comment by CameronNemo on void-packages repository
https://github.com/void-linux/void-packages/pull/38494#issuecomment-1237594555
Comment:
Any word on this? I am trying it out and it seems to work alright, although I guess I have to duplicate the checksums in all my templates.
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PR PATCH] [Closed]: [Draft] xbps-src: clean up fetch hook and allow mirroring
2022-08-06 2:57 [PR PATCH] xbps-src: clean up fetch hook and allow mirroring Duncaen
` (3 preceding siblings ...)
2022-09-06 2:34 ` CameronNemo
@ 2022-10-01 21:29 ` Duncaen
4 siblings, 0 replies; 6+ messages in thread
From: Duncaen @ 2022-10-01 21:29 UTC (permalink / raw)
To: ml
[-- Attachment #1: Type: text/plain, Size: 378 bytes --]
There's a closed pull request on the void-packages repository
[Draft] xbps-src: clean up fetch hook and allow mirroring
https://github.com/void-linux/void-packages/pull/38494
Description:
As an alternative to https://github.com/void-linux/void-packages/pull/38438
Allow mirroring by having multiple distfiles for the same target file name, either by url or by ">filename".
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2022-10-01 21:29 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-08-06 2:57 [PR PATCH] xbps-src: clean up fetch hook and allow mirroring Duncaen
2022-08-07 17:52 ` [Draft] " CameronNemo
2022-08-07 18:09 ` Duncaen
2022-08-07 18:10 ` Duncaen
2022-09-06 2:34 ` CameronNemo
2022-10-01 21:29 ` [PR PATCH] [Closed]: " Duncaen
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).