Improve scraper.sh when running as cron job

Only errors are sent to stderr. An error only happens when a package
version has changed.

lib.sh

@@ -29,7 +29,7 @@ msg2() {
 
 warning() {
 	local mesg=$1; shift
-	printf "${YELLOW}#### $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2
+	printf "${YELLOW}#### $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@"
 }
 
 error() {

push.sh

@@ -87,7 +87,7 @@ if [[ $AZB_BUILD_AUR4 == 1 && $AZB_MODE_GIT == 1 ]]; then
         msg "Packaging $PKG..."
         run_cmd "cd \"$PWD/$PKG\""
         run_cmd "mksrcinfo"
-        run_cmd "git add . && git commit -m 'Update for kernel $AZB_GIT_KERNEL_X64_VERSION_FULL' && git push"
+        run_cmd "git add . && git commit -m 'Update for kernel $AZB_GIT_KERNEL_X64_VERSION_FULL'; git push"
         run_cmd "cd - > /dev/null"
     done
 elif [[ $AZB_BUILD_AUR4 == 1 && $AZB_MODE_LTS == 1 ]]; then

scraper.sh

@@ -37,6 +37,8 @@ for (( a = 0; a < $#; a++ )); do
     fi
 done
 
+CHECK_WEBPAGE_RETVAL=0
+
 check_webpage() {
     # $1: The url to scrape
     # $2: The Perl regex to match with
@@ -46,17 +48,50 @@ check_webpage() {
     debug "Expecting: $3"
     PAGE=""
     if [[ $DEBUG == 1 ]]; then
-        PAGE=$(curl -vsL "${1}")
+        PAGE=$(curl -vsL "${1}"; echo "RETVAL: $?")
     else
-        PAGE=$(curl -sL "${1}")
+        PAGE=$(curl -sL "${1}"; echo "RETVAL: $?")
+    fi
+    if [[ $(echo $PAGE | grep -q "504 Gateway Timeout"; echo $?) == 0 ]]; then
+        # error "IN HERE YO 1"
+        CHECK_WEBPAGE_RETVAL=-1
+        return
+    elif [[ $(echo $PAGE | grep -q "503 Service Unavailable"; echo $?) == 0 ]]; then
+        # error "IN HERE YO 2"
+        CHECK_WEBPAGE_RETVAL=-1
+        return
+    elif [[ $PAGE == "RETVAL: 7" ]]; then
+        # error "IN HERE YO 3"
+        CHECK_WEBPAGE_RETVAL=-1
+        return
     fi
-    debug "Page: ${PAGE}"
+    # debug "Page: ${PAGE}"
     SCRAPED_STRING=$(echo "${PAGE}" | \grep -Po -m 1 "${2}")
     debug "Got \"$SCRAPED_STRING\" from webpage."
     if [[ $SCRAPED_STRING != "$3" ]]; then
+        error "PAGE: $PAGE"
         error "Checking \"$1\" expected \"$3\" got \"$SCRAPED_STRING\""
         debug "Returning 1 from check_webpage()"
-        return 1
+        CHECK_WEBPAGE_RETVAL=1
+        return
+    fi
+    CHECK_WEBPAGE_RETVAL=0
+    return
+}
+
+check_result() {
+    # $1 current line
+    # $2 changed line
+    if [[ $CHECK_WEBPAGE_RETVAL == 0 ]]; then
+        msg2 "The $1 version is current."
+    elif [[ $CHECK_WEBPAGE_RETVAL == 1 ]]; then
+        error "The $2 is out-of-date!"
+        HAS_ERROR=1
+    elif [[ $CHECK_WEBPAGE_RETVAL == -1 ]]; then
+        warning "The $2 package page was unreachable!"
+    else
+        error "Check returned $CHECK_WEBPAGE_RETVAL"
+        HAS_ERROR=1
     fi
 }
 
@@ -73,86 +108,50 @@ msg "scraper.sh started..."
 # Check archiso kernel version (this will change when the archiso is updated)
 #
 msg "Checking archiso download page for linux kernel version changes..."
-
 check_webpage "https://www.archlinux.org/download/" "(?<=Included Kernel:</strong> )[\d\.]+" "$AZB_KERNEL_ARCHISO_VERSION"
-
+check_result "archiso kernel version" "archiso"
-if [[ $? != 0 ]]; then
-    error "The archiso has been changed!"
-    HAS_ERROR=1
-else
-    msg2 "The archiso kernel version is current."
-fi
 
 #
 # Check i686 linux kernel version
 #
 msg "Checking the online package database for i686 linux kernel version changes..."
-
 check_webpage "https://www.archlinux.org/packages/core/i686/linux/" "(?<=<h2>linux )[\d\.-]+(?=</h2>)" "$AZB_GIT_KERNEL_X32_VERSION"
-
+check_result "i686 linux kernel package" "linux i686"
-if [[ $? != 0 ]]; then
-    error "linux i686 is out-of-date!"
-    HAS_ERROR=1
-else
-    msg2 "The i686 linux kernel package is current."
-fi
 
 #
 # Check x86_64 linux kernel version
 #
 msg "Checking the online package database for x86_64 linux kernel version changes..."
-
 check_webpage "https://www.archlinux.org/packages/core/x86_64/linux/" "(?<=<h2>linux )[\d\.-]+(?=</h2>)" "$AZB_GIT_KERNEL_X64_VERSION"
-
+check_result "x86_64 linux kernel package" "linux x86_64"
-if [[ $? != 0 ]]; then
-    error "linux x86_64 is out-of-date!"
-    HAS_ERROR=1
-else
-    msg2 "The x86_64 linux kernel package is current."
-fi
 
 #
 # Check i686 linux-lts kernel version
 #
 msg "Checking the online package database for i686 linux-lts kernel version changes..."
-
 check_webpage "https://www.archlinux.org/packages/core/i686/linux-lts/" "(?<=<h2>linux-lts )[\d\.-]+(?=</h2>)" "$AZB_LTS_KERNEL_X32_VERSION"
-
+check_result "i686 linux-lts kernel package" "linux-lts i686"
-if [[ $? != 0 ]]; then
-    error "linux-lts i686 is out-of-date!"
-    HAS_ERROR=1
-else
-    msg2 "The i686 linux-lts kernel package is current."
-fi
 
 #
 # Check x86_64 linux-lts kernel version
 #
 msg "Checking the online package database for x86_64 linux-lts kernel version changes..."
-
 check_webpage "https://www.archlinux.org/packages/core/x86_64/linux-lts/" "(?<=<h2>linux-lts )[\d\.-]+(?=</h2>)" "$AZB_LTS_KERNEL_X64_VERSION"
-
+check_result "x86_64 linux-lts kernel package" "linux-lts x86_64"
-if [[ $? != 0 ]]; then
-    error "linux-lts x86_64 is out-of-date!"
-    HAS_ERROR=1
-else
-    msg2 "The x86_64 linux-lts kernel package is current."
-fi
 
 #
 # Check ZFSonLinux.org
 #
 msg "Checking zfsonlinux.org for new versions..."
-
 check_webpage "http://zfsonlinux.org/" "(?<=downloads/zfsonlinux/spl/spl-)[\d\.]+(?=.tar.gz)" "$AZB_ZOL_VERSION"
+check_result "ZOL stable version" "ZOL stable version"
 
-if [[ $? != 0 ]]; then
+#
-    error "ZOL version has changed!"
+# This is the end
-    HAS_ERROR=1
+# Beautiful friend
-else
+# This is the end
-    msg2 "The ZOL sources are current."
+# My only friend, the end
-fi
+#
-
 if [[ $HAS_ERROR -eq 1 ]]; then
     exit 1;
 fi