From 95e99adab99eda6b94210ea3cf6af572f3ee05fa Mon Sep 17 00:00:00 2001 From: Jesus Alvarez Date: Sat, 18 Jul 2015 12:56:36 -0700 Subject: [PATCH] Improve scraper.sh when running as cron job Only errors are sent to stderr. An error only happens when a package version has changed. --- lib.sh | 2 +- push.sh | 2 +- scraper.sh | 103 ++++++++++++++++++++++++++--------------------------- 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/lib.sh b/lib.sh index d1e284b..d5c77ca 100644 --- a/lib.sh +++ b/lib.sh @@ -29,7 +29,7 @@ msg2() { warning() { local mesg=$1; shift - printf "${YELLOW}#### $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 + printf "${YELLOW}#### $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" } error() { diff --git a/push.sh b/push.sh index cb48619..a12e771 100755 --- a/push.sh +++ b/push.sh @@ -87,7 +87,7 @@ if [[ $AZB_BUILD_AUR4 == 1 && $AZB_MODE_GIT == 1 ]]; then msg "Packaging $PKG..." run_cmd "cd \"$PWD/$PKG\"" run_cmd "mksrcinfo" - run_cmd "git add . && git commit -m 'Update for kernel $AZB_GIT_KERNEL_X64_VERSION_FULL' && git push" + run_cmd "git add . && git commit -m 'Update for kernel $AZB_GIT_KERNEL_X64_VERSION_FULL'; git push" run_cmd "cd - > /dev/null" done elif [[ $AZB_BUILD_AUR4 == 1 && $AZB_MODE_LTS == 1 ]]; then diff --git a/scraper.sh b/scraper.sh index 4ac7ed5..ded394c 100755 --- a/scraper.sh +++ b/scraper.sh @@ -37,6 +37,8 @@ for (( a = 0; a < $#; a++ )); do fi done +CHECK_WEBPAGE_RETVAL=0 + check_webpage() { # $1: The url to scrape # $2: The Perl regex to match with @@ -46,17 +48,50 @@ check_webpage() { debug "Expecting: $3" PAGE="" if [[ $DEBUG == 1 ]]; then - PAGE=$(curl -vsL "${1}") + PAGE=$(curl -vsL "${1}"; echo "RETVAL: $?") else - PAGE=$(curl -sL "${1}") + PAGE=$(curl -sL "${1}"; echo "RETVAL: $?") + fi + if [[ $(echo $PAGE | grep -q "504 Gateway Timeout"; echo $?) == 0 ]]; then + # error "IN HERE YO 1" + CHECK_WEBPAGE_RETVAL=-1 + return + elif [[ $(echo $PAGE | grep -q "503 Service Unavailable"; echo $?) == 0 ]]; then + # error "IN HERE YO 2" + CHECK_WEBPAGE_RETVAL=-1 + return + elif [[ $PAGE == "RETVAL: 7" ]]; then + # error "IN HERE YO 3" + CHECK_WEBPAGE_RETVAL=-1 + return fi - debug "Page: ${PAGE}" + # debug "Page: ${PAGE}" SCRAPED_STRING=$(echo "${PAGE}" | \grep -Po -m 1 "${2}") debug "Got \"$SCRAPED_STRING\" from webpage." if [[ $SCRAPED_STRING != "$3" ]]; then + error "PAGE: $PAGE" error "Checking \"$1\" expected \"$3\" got \"$SCRAPED_STRING\"" debug "Returning 1 from check_webpage()" - return 1 + CHECK_WEBPAGE_RETVAL=1 + return + fi + CHECK_WEBPAGE_RETVAL=0 + return +} + +check_result() { + # $1 current line + # $2 changed line + if [[ $CHECK_WEBPAGE_RETVAL == 0 ]]; then + msg2 "The $1 version is current." + elif [[ $CHECK_WEBPAGE_RETVAL == 1 ]]; then + error "The $2 is out-of-date!" + HAS_ERROR=1 + elif [[ $CHECK_WEBPAGE_RETVAL == -1 ]]; then + warning "The $2 package page was unreachable!" + else + error "Check returned $CHECK_WEBPAGE_RETVAL" + HAS_ERROR=1 fi } @@ -73,86 +108,50 @@ msg "scraper.sh started..." # Check archiso kernel version (this will change when the archiso is updated) # msg "Checking archiso download page for linux kernel version changes..." - check_webpage "https://www.archlinux.org/download/" "(?<=Included Kernel: )[\d\.]+" "$AZB_KERNEL_ARCHISO_VERSION" - -if [[ $? != 0 ]]; then - error "The archiso has been changed!" - HAS_ERROR=1 -else - msg2 "The archiso kernel version is current." -fi +check_result "archiso kernel version" "archiso" # # Check i686 linux kernel version # msg "Checking the online package database for i686 linux kernel version changes..." - check_webpage "https://www.archlinux.org/packages/core/i686/linux/" "(?<=

linux )[\d\.-]+(?=

)" "$AZB_GIT_KERNEL_X32_VERSION" - -if [[ $? != 0 ]]; then - error "linux i686 is out-of-date!" - HAS_ERROR=1 -else - msg2 "The i686 linux kernel package is current." -fi +check_result "i686 linux kernel package" "linux i686" # # Check x86_64 linux kernel version # msg "Checking the online package database for x86_64 linux kernel version changes..." - check_webpage "https://www.archlinux.org/packages/core/x86_64/linux/" "(?<=

linux )[\d\.-]+(?=

)" "$AZB_GIT_KERNEL_X64_VERSION" - -if [[ $? != 0 ]]; then - error "linux x86_64 is out-of-date!" - HAS_ERROR=1 -else - msg2 "The x86_64 linux kernel package is current." -fi +check_result "x86_64 linux kernel package" "linux x86_64" # # Check i686 linux-lts kernel version # msg "Checking the online package database for i686 linux-lts kernel version changes..." - check_webpage "https://www.archlinux.org/packages/core/i686/linux-lts/" "(?<=

linux-lts )[\d\.-]+(?=

)" "$AZB_LTS_KERNEL_X32_VERSION" - -if [[ $? != 0 ]]; then - error "linux-lts i686 is out-of-date!" - HAS_ERROR=1 -else - msg2 "The i686 linux-lts kernel package is current." -fi +check_result "i686 linux-lts kernel package" "linux-lts i686" # # Check x86_64 linux-lts kernel version # msg "Checking the online package database for x86_64 linux-lts kernel version changes..." - check_webpage "https://www.archlinux.org/packages/core/x86_64/linux-lts/" "(?<=

linux-lts )[\d\.-]+(?=

)" "$AZB_LTS_KERNEL_X64_VERSION" - -if [[ $? != 0 ]]; then - error "linux-lts x86_64 is out-of-date!" - HAS_ERROR=1 -else - msg2 "The x86_64 linux-lts kernel package is current." -fi +check_result "x86_64 linux-lts kernel package" "linux-lts x86_64" # # Check ZFSonLinux.org # msg "Checking zfsonlinux.org for new versions..." - check_webpage "http://zfsonlinux.org/" "(?<=downloads/zfsonlinux/spl/spl-)[\d\.]+(?=.tar.gz)" "$AZB_ZOL_VERSION" +check_result "ZOL stable version" "ZOL stable version" -if [[ $? != 0 ]]; then - error "ZOL version has changed!" - HAS_ERROR=1 -else - msg2 "The ZOL sources are current." -fi - +# +# This is the end +# Beautiful friend +# This is the end +# My only friend, the end +# if [[ $HAS_ERROR -eq 1 ]]; then exit 1; fi