From 24815efefc6375b558904d4c54a1f533d8a6a80c Mon Sep 17 00:00:00 2001
From: Dalibor Pospisil <dapospis@redhat.com>
Date: Tue, 16 May 2017 17:38:51 +0200
Subject: [PATCH] reworked fallback mechanism, fixing bz1448510
---
src/rpms.sh | 251 ++++++++++++++++++++++++++++++++++++++++++------------------
1 file changed, 177 insertions(+), 74 deletions(-)
diff --git a/src/rpms.sh b/src/rpms.sh
index 152ba84..9bd1561 100644
--- a/src/rpms.sh
+++ b/src/rpms.sh
@@ -565,93 +565,188 @@ from https://kojipkgs.fedoraproject.org/packages.
# return information of the first matching package
+# $1 - method (rpm | repoquery)
+# $2 - packages (NVR)
__INTERNAL_rpmGetPackageInfo() {
- local package_info package_info_raw res=0
- rlLogDebug "${FUNCNAME}(): getting package info for '$1'"
- package_info_raw="$(rpm --qf "%{name} %{version} %{release} %{arch} %{sourcerpm}\n" -q "$1")" || {
- rlLogDebug "${FUNCNAME}(): package_info_raw: '$package_info_raw'"
- rlLogInfo "rpm '$1' not installed, trying repoquery"
- package_info_raw="$(repoquery --qf "%{name} %{version} %{release} %{arch} %{sourcerpm}\n" -q "$1")"
- } || {
- rlLogDebug "${FUNCNAME}(): package_info_raw: '$package_info_raw'"
- rlLogError "rpm '$1' not available in any repository"
- let res++
+ local package_info package_info_raw res=0
+ rlLogDebug "${FUNCNAME}(): getting package info for '$2'"
+ package_info_raw="$($1 --qf "%{name} %{version} %{release} %{arch} %{sourcerpm}\n" -q "$2")"
+ [[ $? -ne 0 || -z "$package_info_raw" ]] && {
+ rlLogDebug "${FUNCNAME}(): package_info_raw: '$package_info_raw'"
+ rlLogInfo "rpm '$2' not available using '$1' tool"
+ let res++
+ }
+ if [[ $res -eq 0 ]]; then
+ rlLogDebug "${FUNCNAME}(): package_info_raw: '$package_info_raw'"
+ # get first package
+ local tmp=( $(echo "$package_info_raw" | head -n 1 ) )
+ # extract component name from source_rpm
+ package_info="${tmp[@]:0:4} $(__INTERNAL_getNVRA "${tmp[4]/.rpm}")" || {
+ rlLogError "parsing package info '$package_info_raw' failed"
+ let res++
}
- if [[ $res -eq 0 ]]; then
- rlLogDebug "${FUNCNAME}(): package_info_raw: '$package_info_raw'"
- # get first package
- local tmp=( $(echo "$package_info_raw" | head -n 1 ) )
- # extract component name from source_rpm
- package_info="${tmp[@]:0:4} $(__INTERNAL_getNVRA "${tmp[4]/.rpm}")" || {
- rlLogError "parsing package info '$package_info_raw' failed"
- let res++
- }
- rlLogDebug "${FUNCNAME}(): package_info: '$package_info'"
- echo "$package_info"
- fi
- rlLogDebug "${FUNCNAME}(): returning $res"
- return $res
+ rlLogDebug "${FUNCNAME}(): package_info: '$package_info'"
+ rlLogInfo "got rpm info for '$2' via '$1'"
+ echo "$package_info"
+ fi
+ rlLogDebug "${FUNCNAME}(): returning $res"
+ return $res
}
-__INTERNAL_getNVRA() {
- rlLogDebug "${FUNCNAME}(): parsing NVRA for '$1'"
- local pat='(.+)-([^-]+)-(.+)\.([^.]+)$'
- [[ "$1" =~ $pat ]] && echo "${BASH_REMATCH[@]:1}"
+BEAKERLIB_rpm_fetch_base_url=( "https://kojipkgs.fedoraproject.org/packages" )
+BEAKERLIB_rpm_packageinfo_base_url=( "http://koji.fedoraproject.org/koji" )
+
+# generate combinations for various methods and parameters of finding the package
+__INTERNAL_rpmInitUrl() {
+ local i j k
+
+ rlLogDebug "${FUNCNAME}(): preparing download variants"
+
+ __INTERNAL_rpmGetNextUrl_phase=()
+ for k in rpm repoquery; do
+ for j in nvr n; do
+ for i in "${BEAKERLIB_rpm_fetch_base_url[@]}"; do
+ __INTERNAL_rpmGetNextUrl_phase+=( "$k" ); # tool
+ __INTERNAL_rpmGetNextUrl_phase+=( "$j" ); # package_spec
+ __INTERNAL_rpmGetNextUrl_phase+=( "$i" ); # base_url
+ done
+ done
+ done
+ for j in koji; do
+ for i in "${BEAKERLIB_rpm_packageinfo_base_url[@]}"; do
+ __INTERNAL_rpmGetNextUrl_phase+=( "$j" ) ; # tool
+ __INTERNAL_rpmGetNextUrl_phase+=( "nvra.rpm" ) ; # package_spec
+ __INTERNAL_rpmGetNextUrl_phase+=( "$i" ) ; # base_url
+ done
+ done
+ rlLogDebug "${FUNCNAME}(): $(set | grep ^__INTERNAL_rpmGetNextUrl_phase=)"
}
-__INTERNAL_rpmGetPath() {
+__INTERNAL_WGET="wget -t 3 -T 180 -w 20 --waitretry=30 --no-check-certificate"
+
+# __INTERNAL_rpmGetNextUrl N V R A | --source N V R
+__INTERNAL_rpmGetNextUrl() {
local source=''
[[ "$1" == "--source" ]] && {
source="$1"
shift
}
- local package="$1"
- local N V R A CN CV CR CA res=0 nil
- if IFS=' ' read N V R A < <(__INTERNAL_getNVRA "$package"); then
- rlLogDebug "${FUNCNAME}(): getting component for $N-$V-$R.$A"
- IFS=' ' read nil nil nil nil CN CV CR CA < <(__INTERNAL_rpmGetPackageInfo "$N") || \
- let res++
- else
- rlLogDebug "${FUNCNAME}(): getting component, N, V, R, and A for $1"
- IFS=' ' read N V R A CN CV CR CA < <(__INTERNAL_rpmGetPackageInfo "$package") || \
- let res++
- fi
- if [[ $res -eq 0 ]]; then
- if [[ -z "$source" ]]; then
- echo "$CN/$CV/$CR/$A/$N-$V-$R.$A.rpm"
- else
- echo "$CN/$CV/$CR/src/$CN-$CV-$CR.src.rpm"
- fi
- else
- rlLogError "rpm path counld not be constructed"
- fi
+ local N=$1 V=$2 R=$3 A=$4 nil res url
+ rlLogDebug "${FUNCNAME}(): process $N-$V-$R.$A"
+ while [[ -n "$__INTERNAL_rpmGetNextUrl_phase" ]]; do
+ res=0
+ local tool=${__INTERNAL_rpmGetNextUrl_phase[0]}
+ local package_spec=${__INTERNAL_rpmGetNextUrl_phase[1]}
+ local base_url=${__INTERNAL_rpmGetNextUrl_phase[2]}
+ #rlLogDebug "${FUNCNAME}(): $(set | grep ^__INTERNAL_rpmGetNextUrl_phase=)"
+ rlLogDebug "${FUNCNAME}(): remove first three indices of __INTERNAL_rpmGetNextUrl_phase"
+ __INTERNAL_rpmGetNextUrl_phase=( "${__INTERNAL_rpmGetNextUrl_phase[@]:3}" )
+ rlLogInfo "trying tool $tool with $package_spec"
+ case $tool,$package_spec in
+ *,nvr)
+ IFS=' ' read nil nil nil nil CN CV CR CA < <(__INTERNAL_rpmGetPackageInfo $tool "$N-$V-$R")
+ res=$?
+ if [[ -n "$source" ]]; then
+ url="$base_url/$CN/$CV/$CR/src/$CN-$CV-$CR.src.rpm"
+ else
+ url="$base_url/$CN/$CV/$CR/$A/$N-$V-$R.$A.rpm"
+ fi
+ ;;
+ *,n)
+ IFS=' ' read nil nil nil nil CN CV CR CA < <(__INTERNAL_rpmGetPackageInfo $tool "$N")
+ res=$?
+ if [[ -n "$source" ]]; then
+ url="$base_url/$CN/$V/$R/src/$CN-$V-$R.src.rpm"
+ else
+ url="$base_url/$CN/$V/$R/$A/$N-$V-$R.$A.rpm"
+ fi
+ ;;
+ koji,nvra.rpm)
+ rlLogDebug "$FUNCNAME(): get rpm info"
+ local rpm_info=$($__INTERNAL_WGET -O - "$base_url/search?match=exact&type=rpm&terms=$N-$V-$R.$A.rpm")
+ [[ $? -ne 0 || -z "$rpm_info" ]] && {
+ rlLogError "could not download rpm information"
+ let res++
+ continue
+ }
+ #[[ "$DEBUG" ]] && rlLogDebug "rpm_info='$rpm_info'"
+ local buildurl=$(echo "$rpm_info" | grep Version | grep -o '[^"]*buildID=[^"]*')
+ [[ $? -ne 0 || -z "$buildurl" ]] && {
+ rlLogError "could not find buildID"
+ let res++
+ continue
+ }
+ rlLogDebug "$FUNCNAME(): extracted buildurl='$buildurl'"
+ [[ "$buildurl" =~ http ]] || buildurl="$base_url/$buildurl"
+ rlLogDebug "$FUNCNAME(): using buildurl='$buildurl'"
+ local buildinfo=$($__INTERNAL_WGET -O - "$buildurl")
+ [[ $? -ne 0 || -z "$buildinfo" ]] && {
+ rlLogError "could not download build information"
+ let res++
+ continue
+ }
+ #[[ -n "$DEBUG" ]] && rlLogDebug "buildinfo='$buildinfo'"
+ if [[ -n "$source" ]]; then
+ url=$(echo "$buildinfo" | grep download | grep -o 'http[^"]*.src.rpm')
+ else
+ url=$(echo "$buildinfo" | grep download | grep -o "http[^\"]*/$N-$V-$R.$A.rpm")
+ fi
+ [[ $? -ne 0 ]] && {
+ rlLogError "could not find package url"
+ let res++
+ continue
+ }
+ ;;
+ *)
+ rlLogDebug "$FUNCNAME(): unknown case"
+ rlLogError "there's a bug in the code, unknown case!"
+ let res++
+ break
+ esac
+ [[ $res -eq 0 ]] && break
+ done
+
+ [[ -z "$url" ]] && {
+ rlLogError "could not find package url"
+ let res++
+ }
+
+ rlLogDebug "$FUNCNAME(): using url='$url'"
+
+ [[ $res -eq 0 ]] && __INTERNAL_RETURN_VALUE="$url"
+
rlLogDebug "${FUNCNAME}(): returning $res"
return $res
}
-BEAKERLIB_rpm_fetch_base_url=( "https://kojipkgs.fedoraproject.org/packages" )
+__INTERNAL_getNVRA() {
+ rlLogDebug "${FUNCNAME}(): parsing NVRA for '$1'"
+ local pat='(.+)-([^-]+)-(.+)\.([^.]+)$'
+ [[ "$1" =~ $pat ]] && echo "${BASH_REMATCH[@]:1}"
+}
+
__INTERNAL_rpmDirectDownload() {
- local res=0 url pkg baseurl path
- path="$(__INTERNAL_rpmGetPath "$@")" || return 1
- for baseurl in "${BEAKERLIB_rpm_fetch_base_url[@]}"; do
- url="$baseurl/$path"
+ local url pkg
+ __INTERNAL_rpmInitUrl
+ while __INTERNAL_rpmGetNextUrl "$@"; do
+ url="$__INTERNAL_RETURN_VALUE"; unset __INTERNAL_RETURN_VALUE
local pkg=$(basename "$url")
rlLog "trying download from '$url'"
- if wget --no-check-certificate -O $pkg "$url" >&2; then
- rlLogDebug "$FUNCNAME(): package '$pkg' was successfully downloaded"
+ if $__INTERNAL_WGET -O $pkg "$url"; then
+ rlLogInfo "$FUNCNAME(): package '$pkg' was successfully downloaded"
echo "$pkg"
return 0
- else
- rlLogDebug "$FUNCNAME(): package '$pkg' was not successfully downloaded"
- let res++
fi
+ rm -f "$pkg"
+ rlLogWarning "package '$pkg' was not successfully downloaded"
done
- rlLogDebug "${FUNCNAME}(): returning $res"
- return $res
+ rlLogError "package '$pkg' was not successfully downloaded"
+ rlLogDebug "${FUNCNAME}(): returning 1"
+ return 1
}
@@ -659,15 +754,16 @@ __INTERNAL_rpmDirectDownload() {
# __INTERNAL_rpmGetWithYumDownloader
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
-# Download package(s) using yumdownloader
+# Download package using yumdownloader
__INTERNAL_rpmGetWithYumDownloader() {
local source=''
+ local package="$1-$2-$3.$4"
[[ "$1" == "--source" ]] && {
source="$1"
shift
+ local package="$1-$2-$3"
}
- local package="$1" # list of packages to download
rlLogDebug "${FUNCNAME}(): Trying yumdownloader to download $package"
if ! which yumdownloader &> /dev/null ; then
@@ -697,6 +793,7 @@ __INTERNAL_rpmGetWithYumDownloader() {
return 1
}
[[ $pkg_cnt -gt 1 ]] && rlLogWarning "got more than one package"
+ rm -f $pkg
rlLogDebug "$FUNCNAME(): moving package to local dir"
mv $tmp/* ./
rlLogDebug "$FUNCNAME(): removing tmp dir '$tmp'"
@@ -811,7 +908,7 @@ rlRpmInstall(){
return 0
else
local tmp=$(mktemp -d)
- ( cd $tmp; __INTERNAL_rpmDownload "$N-$V-$R.$A" )
+ ( cd $tmp; __INTERNAL_rpmDownload $N $V $R $A )
if [ $? -eq 0 ]; then
rlLog "RPM: $N-$V-$R.$A.rpm"
rpm -Uhv --oldpackage "$tmp/$N-$V-$R.$A.rpm"
@@ -870,7 +967,7 @@ Returns 0 if specified package was downloaded succesfully.
=cut
rlRpmDownload(){
- local source='' NVRA res pkg
+ local source='' res pkg N V R A
[[ "$1" == "--source" ]] && {
source="$1"
shift
@@ -878,23 +975,28 @@ rlRpmDownload(){
if [[ $# -eq 1 ]]; then
local package="$1"
[[ -n "$source" ]] && package="$package.src"
- if ! __INTERNAL_getNVRA "$package" > /dev/null; then
+ if ! IFS=' ' read N V R A < <(__INTERNAL_getNVRA "$package"); then
rlLogError "$FUNCNAME: Bad N.V.R-A format"
return 1
fi
- NVRA="$1"
elif [[ -z "$source" && $# -eq 4 ]]; then
- NVRA="$1-$2-$3.$4"
+ N="$1"
+ V="$2"
+ R="$3"
+ A="$4"
elif [[ -n "$source" && $# -ge 3 ]]; then
- NVRA="$1-$2-$3"
+ N="$1"
+ V="$2"
+ R="$3"
+ A=""
else
rlLogError "$FUNCNAME: invalid parameter(s)"
return 1
fi
- rlLog "$FUNCNAME: Fetching ${source:+source }RPM $NVRA"
+ rlLog "$FUNCNAME: Fetching ${source:+source }RPM $N-$V-$R.$A"
- if pkg=$(__INTERNAL_rpmDownload $source "$NVRA"); then
+ if pkg=$(__INTERNAL_rpmDownload $source $N $V $R $A); then
rlLog "RPM: $pkg"
echo "$pkg"
return 0
@@ -932,13 +1034,14 @@ Returns 0 if the source package was succesfully downloaded.
rlFetchSrcForInstalled(){
local PKGNAME=$1 srcrpm
- if ! PKG=$(rpm -q ${PKGNAME}); then
+ local N V R nil
+ if ! IFS=' ' read N V R nil nil nil nil nil < <((__INTERNAL_rpmGetPackageInfo rpm "$PKGNAME")); then
rlLogError "$FUNCNAME: The package is not installed, can't download the source"
return 1
fi
- rlLog "$FUNCNAME: Fetching source rpm for installed $PKG"
+ rlLog "$FUNCNAME: Fetching source rpm for installed $N-$V-$R"
- if srcrpm="$(__INTERNAL_rpmDownload --source "$PKG")"; then
+ if srcrpm="$(__INTERNAL_rpmDownload --source $N $V $R)"; then
echo "$srcrpm"
return 0
else
--
2.9.3