Refactoring for standard packages is done

pull/57/merge
Jesus Alvarez 9 years ago
parent f74ff01064
commit 97419b1239
  1. 108
      build.sh
  2. 16
      conf.sh
  3. 54
      lib.sh
  4. 44
      push.sh
  5. 295
      repo.sh
  6. 8
      scraper.sh
  7. 75
      verifier.sh

@ -8,20 +8,7 @@
#
# Defaults, don't edit these.
AZB_UPDATE_PKGBUILDS=""
AZB_UPDPKGSUMS=0
AZB_UPDATE_TEST_PKGBUILDS=""
AZB_BUILD=0
AZB_USE_TEST=0
AZB_CHROOT_UPDATE=""
AZB_SIGN=""
AZB_CLEANUP=0
AZB_MODE_DEF=0
AZB_MODE_GIT=0
AZB_MODE_LTS=0
NAME=$(basename $0)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@ -36,14 +23,34 @@ fi
if ! source ${SCRIPT_DIR}/src/HEADER.sh; then
error "Could not load ./src/HEADER.sh!"
error "Could not load src/HEADER.sh!"
fi
trap 'trap_abort' INT QUIT TERM HUP
trap 'trap_exit' EXIT
# Defaults, don't edit these.
AZB_UPDATE_PKGBUILDS=""
AZB_UPDPKGSUMS=0
AZB_UPDATE_TEST_PKGBUILDS=""
AZB_BUILD=0
AZB_USE_TEST=0
AZB_CHROOT_UPDATE=""
AZB_SIGN=""
AZB_CLEANUP=0
AZB_COMMAND=""
AZB_MODE=""
AZB_MODE_STD=0
AZB_MODE_GIT=0
AZB_MODE_LTS=0
usage() {
echo "build.sh - A build script for archzfs"
echo "${NAME} - A build script for archzfs"
echo
echo "Usage: build.sh [options] [mode] [command [command option] [...]"
echo "Usage: ${NAME} [options] mode command [command ...]"
echo
echo "Options:"
echo
@ -56,9 +63,9 @@ usage() {
echo
echo "Modes:"
echo
echo " def Use default packages."
echo " git Use the git packages."
echo " lts Use the lts packages."
echo " std Build the standard packages."
echo " git Build the git packages."
echo " lts Build the lts packages."
echo
echo "Commands:"
echo
@ -70,23 +77,19 @@ usage() {
echo
echo "Examples:"
echo
echo " build.sh -C :: Remove all compiled packages"
echo " build.sh git make -u :: Update the chroot and build all of the packages"
echo " build.sh lts update :: Update PKGBUILDS only"
echo " build.sh git update make -u :: Update PKGBUILDs, update the chroot, and make all of the packages"
echo " build.sh lts update-test test -u :: Update PKGBUILDs (use testing versions), update the chroot, and make all of the packages"
echo " ${NAME} -C :: Remove all compiled packages"
echo " ${NAME} git make -u :: Update the chroot and build all of the packages"
echo " ${NAME} lts update :: Update PKGBUILDS only"
echo " ${NAME} git update make -u :: Update PKGBUILDs, update the chroot, and make all of the packages"
echo " ${NAME} lts update-test test -u :: Update PKGBUILDs (use testing versions), update the chroot, and make all of the packages"
trap - EXIT # Prevents exit log output
}
trap 'trap_abort' INT QUIT TERM HUP
trap 'trap_exit' EXIT
build_sources() {
for PKG in ${AZB_PKG_LIST}; do
msg "Building source for $PKG";
run_cmd "cd \"$PWD/packages/$PKG\" && mkaurball -f"
run_cmd "cd \"$PWD/packages/${AZB_MODE}/$PKG\" && mkaurball -f"
done
}
@ -173,19 +176,19 @@ check_git_repo() {
update_def_pkgbuilds() {
AZB_KERNEL_VERSION_FULL=$(full_kernel_version ${AZB_DEF_KERNEL_VERSION})
AZB_KERNEL_VERSION_FULL=$(full_kernel_version ${AZB_STD_KERNEL_VERSION})
AZB_KERNEL_MOD_PATH="${AZB_KERNEL_VERSION_FULL}-ARCH"
AZB_ARCHZFS_PACKAGE_GROUP="archzfs"
AZB_PKGVER=${AZB_ZOL_VERSION}_$(full_kernel_version_no_hyphen ${AZB_DEF_KERNEL_VERSION})
AZB_PKGREL=${AZB_DEF_PKGREL}
AZB_PKGVER=${AZB_ZOL_VERSION}_$(full_kernel_version_no_hyphen ${AZB_STD_KERNEL_VERSION})
AZB_PKGREL=${AZB_STD_PKGREL}
AZB_SPL_UTILS_PKGNAME="spl-utils"
AZB_SPL_PKGNAME="spl"
AZB_ZFS_UTILS_PKGNAME="zfs-utils"
AZB_ZFS_PKGNAME="zfs"
AZB_SPL_UTILS_PKGBUILD_PATH="packages/spl-utils"
AZB_SPL_PKGBUILD_PATH="packages/spl"
AZB_ZFS_UTILS_PKGBUILD_PATH="packages/zfs-utils"
AZB_ZFS_PKGBUILD_PATH="packages/zfs"
AZB_SPL_UTILS_PKGBUILD_PATH="packages/${AZB_MODE}/spl-utils"
AZB_SPL_PKGBUILD_PATH="packages/${AZB_MODE}/spl"
AZB_ZFS_UTILS_PKGBUILD_PATH="packages/${AZB_MODE}/zfs-utils"
AZB_ZFS_PKGBUILD_PATH="packages/${AZB_MODE}/zfs"
debug "AZB_HEADER: ${AZB_HEADER}"
debug "AZB_PKGVER: ${AZB_PKGVER}"
@ -257,7 +260,7 @@ update_lts_pkgbuilds() {
build_packages() {
for PKG in ${AZB_PKG_LIST}; do
msg "Building $PKG..."
run_cmd "cd \"$PWD/packages/$PKG\" && sudo ccm64 s"
run_cmd "cd \"$PWD/packages/${AZB_MODE}/$PKG\" && sudo ccm64 s"
msg2 "${PKG} package files:"
run_cmd "tree ${AZB_CHROOT_PATH}/build/${PKG}/pkg"
done
@ -266,6 +269,7 @@ build_packages() {
run_cmd "find . -iname \"*.log\" -print -exec rm {} \\;"
}
if [[ $# -lt 1 ]]; then
usage;
exit 0;
@ -274,28 +278,38 @@ fi
ARGS=("$@")
for (( a = 0; a < $#; a++ )); do
if [[ ${ARGS[$a]} == "def" ]]; then
AZB_MODE_DEF=1
if [[ ${ARGS[$a]} == "std" ]]; then
AZB_MODE_STD=1
AZB_MODE="std"
elif [[ ${ARGS[$a]} == "git" ]]; then
AZB_MODE_GIT=1
AZB_MODE="git"
elif [[ ${ARGS[$a]} == "lts" ]]; then
AZB_MODE_LTS=1
AZB_MODE="lts"
elif [[ ${ARGS[$a]} == "make" ]]; then
AZB_BUILD=1
AZB_COMMAND="make"
elif [[ ${ARGS[$a]} == "test" ]]; then
AZB_USE_TEST=1
AZB_COMMAND="test"
elif [[ ${ARGS[$a]} == "update" ]]; then
AZB_UPDATE_PKGBUILDS=1
AZB_COMMAND="update"
elif [[ ${ARGS[$a]} == "update-test" ]]; then
AZB_UPDATE_TEST_PKGBUILDS=1
AZB_COMMAND="update-test"
elif [[ ${ARGS[$a]} == "sign" ]]; then
AZB_SIGN=1
AZB_COMMAND="sign"
elif [[ ${ARGS[$a]} == "-u" ]]; then
AZB_CHROOT_UPDATE="-u"
elif [[ ${ARGS[$a]} == "-U" ]]; then
AZB_UPDPKGSUMS=1
AZB_COMMAND="update-pkgsums"
elif [[ ${ARGS[$a]} == "-C" ]]; then
AZB_CLEANUP=1
AZB_COMMAND="clean"
elif [[ ${ARGS[$a]} == "-n" ]]; then
DRY_RUN=1
elif [[ ${ARGS[$a]} == "-d" ]]; then
@ -316,16 +330,16 @@ if [[ $AZB_CLEANUP -eq 1 && $# -gt 1 ]]; then
fi
if [[ ${AZB_MODE_DEF} -eq 0 && ${AZB_MODE_GIT} -eq 0 && ${AZB_MODE_LTS} -eq 0 && $AZB_CLEANUP -eq 0 ]]; then
echo -e "\n"
error "A build mode must be selected!"
echo -e "\n"
if [[ ${AZB_MODE} == "" || ${AZB_COMMAND} == "" ]]; then
echo
error "A build mode and command must be selected!"
echo
usage;
exit 0;
fi
msg "$(date) :: build.sh started..."
msg "$(date) :: ${NAME} started..."
if [[ $AZB_UPDPKGSUMS -eq 1 && ${AZB_MODE_LTS} -eq 1 ]]; then
@ -340,11 +354,11 @@ fi
AZB_PKG_LIST=""
if [[ ${AZB_UPDATE_PKGBUILDS} -eq 1 && ${AZB_MODE_DEF} -eq 1 ]]; then
if [[ ${AZB_UPDATE_PKGBUILDS} -eq 1 && ${AZB_MODE_STD} -eq 1 ]]; then
msg "Updating default pkgbuilds"
update_def_pkgbuilds
if [[ ${AZB_BUILD} -eq 1 ]]; then
AZB_PKG_LIST=${AZB_DEF_PKG_LIST}
AZB_PKG_LIST=${AZB_STD_PKG_LIST}
build_packages
fi
elif [[ ${AZB_UPDATE_PKGBUILDS} -eq 1 && ${AZB_MODE_GIT} -eq 1 ]]; then

@ -10,14 +10,14 @@ AZB_ZFS_INITCPIO_INSTALL_HASH="1e20071fa61a33874505dae0f2d71bb560f43e7faaea735cb
AZB_ZFS_INITCPIO_HOOK_HASH="438a1399d1df5ef20eff37b4d775595fae9943d0c5c105c7bc286b2babcd759e"
# Kernel versions for default ZFS packages
AZB_DEF_PKGREL="1"
AZB_DEF_KERNEL_VERSION="4.5-1"
AZB_STD_PKGREL="1"
AZB_STD_KERNEL_VERSION="4.5-1"
# Kernel version for GIT packages
AZB_DEF_GIT_SPL_COMMIT="5079f5b3"
AZB_DEF_GIT_ZFS_COMMIT="21f21fe8"
AZB_DEF_GIT_PKGREL="1"
AZB_DEF_GIT_KERNEL_VERSION="4.5-1"
AZB_GIT_SPL_COMMIT="5079f5b3"
AZB_GIT_ZFS_COMMIT="21f21fe8"
AZB_GIT_PKGREL="1"
AZB_GIT_KERNEL_VERSION="4.5-1"
# Kernel versions for LTS packages
AZB_LTS_PKGREL="1"
@ -25,7 +25,7 @@ AZB_LTS_KERNEL_VERSION="4.4.7-1"
# Archiso Configuration
AZB_ARCHISO_PKGREL="1"
AZB_KERNEL_ARCHISO_VERSION="4.4.5-1"
AZB_ARCHISO_KERNEL_VERSION="4.4.5"
# Testing repo Linux version dependencies
# AZB_KERNEL_TEST_VERSION="3.13.8-1"
@ -47,6 +47,6 @@ AZB_CHROOT_PATH="/opt/chroot/x86_64/$(whoami)"
# Package backup directory (for adding packages to demz-repo-archiso)
AZB_PACKAGE_BACKUP_DIR="/data/pacman/repo/archive_archzfs"
AZB_DEF_PKG_LIST="spl-utils spl zfs-utils zfs"
AZB_STD_PKG_LIST="spl-utils spl zfs-utils zfs"
AZB_GIT_PKG_LIST="spl-utils-git spl-git zfs-utils-git zfs-git"
AZB_LTS_PKG_LIST="spl-utils-lts spl-lts zfs-utils-lts zfs-lts"

@ -30,11 +30,7 @@ plain() {
plain_one_line() {
local mesg=$1; shift
printf "${ALL_OFF}${ALL_OFF} %s\n\n" "${mesg}"
if [[ $# -gt 0 ]]; then
printf '%s ' "${@}"
printf '\n\n'
fi
printf "${ALL_OFF}${ALL_OFF}%s %s\n\n" "${mesg}" "${@}"
}
@ -162,7 +158,7 @@ norun() {
local mesg=$1; shift
printf "${MAGENTA}XXXX NORUN: ${ALL_OFF}${BOLD}${mesg}${ALL_OFF}\n\n" "$mesg"
if [[ $# -gt 0 ]]; then
printf '%s ' $@
printf '%s ' "$@"
printf '\n\n'
fi
}
@ -188,13 +184,14 @@ run_cmd() {
# Runs a command, capture the output in RUN_CMD_OUTPUT, but also show stdout.
# To use this function, define the following in your calling script:
# RUN_CMD_OUTPUT=""
# RUN_CMD_RETURN=""
run_cmd_show_and_capture_output() {
# $@: The command and args to run
if [[ $DRY_RUN -eq 1 ]]; then
norun "CMD:" $@
norun "CMD:" "$@"
else
plain "Running command:" $@
plain "Running command:" "$@"
plain_one_line "Output:"
# The following allows command output to be displayed in jenkins and stored in the variable simultaneously
# http://www.tldp.org/LDP/abs/html/x17974.html
@ -213,6 +210,30 @@ run_cmd_show_and_capture_output() {
}
# Runs a command, capture the output in RUN_CMD_OUTPUT, but also show stdout. Ignores DRY_RUN=1.
# To use this function, define the following in your calling script:
# RUN_CMD_OUTPUT=""
# RUN_CMD_RETURN=""
run_cmd_show_and_capture_output_no_dry_run() {
# $@: The command and args to run
plain "Running command:" "$@"
plain_one_line "Output:"
# The following allows command output to be displayed in jenkins and stored in the variable simultaneously
# http://www.tldp.org/LDP/abs/html/x17974.html
# WARNING: This function sometimes results in the following error:
# lib.sh: line 145: /usr/bin/tee: Argument list too long
# lib.sh: line 145: /bin/cat: Argument list too long
exec 6>&1 # Link file descriptor 6 with stdout.
RUN_CMD_OUTPUT=$(echo -e "$@" | source /dev/stdin | tee >(cat - >&6); exit ${PIPESTATUS[1]})
exec 1>&6 6>&- # Restore stdout and close file descriptor #6.
RUN_CMD_RETURN=$?
echo
plain_one_line "Command returned:" "${RUN_CMD_RETURN}"
}
# Runs the command, does not show output to stdout
# To use this function, define the following in your calling script:
# RUN_CMD_OUTPUT=""
@ -220,16 +241,29 @@ run_cmd_show_and_capture_output() {
run_cmd_no_output() {
# $@: The command and args to run
if [[ $DRY_RUN -eq 1 ]]; then
norun "CMD:" $@
norun "CMD:" "$@"
else
plain "Running command:" "$@"
RUN_CMD_OUTPUT=$(printf "$@" | source /dev/stdin)
RUN_CMD_OUTPUT=$(echo -e "$@" | source /dev/stdin)
RUN_CMD_RETURN=$?
plain_one_line "Command returned:" "${RUN_CMD_RETURN}"
fi
}
# Runs the command, does not show output to stdout, ignores DRY_RUN=1
# To use this function, define the following in your calling script:
# RUN_CMD_OUTPUT=""
# RUN_CMD_RETURN=""
run_cmd_no_output_no_dry_run() {
# $@: The command and args to run
plain "Running command:" "$@"
RUN_CMD_OUTPUT=$(echo -e "$@" | source /dev/stdin)
RUN_CMD_RETURN=$?
plain_one_line "Command returned:" "${RUN_CMD_RETURN}"
}
package_arch_from_path() {
# $1: Package path
pacman -Qip "$1" | grep "Architecture" | cut -d : -f 2 | tr -d ' '

@ -27,11 +27,12 @@ trap 'trap_exit' EXIT
DRY_RUN=0 # Show commands only. Don't do anything.
DEBUG=0 # Show debug output.
AZB_MODE_DEF=0
AZB_MODE=""
AZB_MODE_STD=0
AZB_MODE_GIT=0
AZB_MODE_LTS=0
AZB_PKG_LIST=""
AZB_KERNEL_VERSION=""
usage() {
@ -47,7 +48,7 @@ usage() {
echo
echo "Modes:"
echo
echo " def Use the default packages."
echo " std Use the standard packages."
echo " git Use the git packages."
echo " lts Use the lts packages."
echo
@ -61,12 +62,15 @@ usage() {
ARGS=("$@")
for (( a = 0; a < $#; a++ )); do
if [[ ${ARGS[$a]} == "def" ]]; then
AZB_MODE_DEF=1
if [[ ${ARGS[$a]} == "std" ]]; then
AZB_MODE_STD=1
AZB_MODE="std"
elif [[ ${ARGS[$a]} == "git" ]]; then
AZB_MODE_GIT=1
AZB_MODE="git"
elif [[ ${ARGS[$a]} == "lts" ]]; then
AZB_MODE_LTS=1
AZB_MODE="lts"
elif [[ ${ARGS[$a]} == "-n" ]]; then
DRY_RUN=1
elif [[ ${ARGS[$a]} == "-d" ]]; then
@ -78,7 +82,13 @@ for (( a = 0; a < $#; a++ )); do
done
if [[ ${AZB_MODE_DEF} -eq 0 && ${AZB_MODE_GIT} -eq 0 && ${AZB_MODE_LTS} -eq 0 ]]; then
if [[ $# -lt 1 ]]; then
usage;
exit 0;
fi
if [[ ${AZB_MODE_STD} -eq 0 && ${AZB_MODE_GIT} -eq 0 && ${AZB_MODE_LTS} -eq 0 ]]; then
echo
error "A mode must be selected!"
usage;
@ -86,29 +96,31 @@ if [[ ${AZB_MODE_DEF} -eq 0 && ${AZB_MODE_GIT} -eq 0 && ${AZB_MODE_LTS} -eq 0 ]]
fi
AZB_PKG_LIST=""
msg "$(date) :: ${NAME} started..."
push_packages() {
for PKG in ${AZB_PKG_LIST}; do
full_kernel_git_version
msg "Packaging ${PKG}..."
run_cmd "cd \"${PWD}/${PKG}\""
run_cmd "mksrcinfo"
run_cmd "git add . && git commit -m 'Update for kernel $(full_kernel_version ${AZB_DEF_GIT_KERNEL_VERSION})'"
run_cmd "git push"
run_cmd "cd - > /dev/null"
local cmd="cd \"${PWD}/packages/${AZB_MODE}/${PKG}\" && "
cmd+="mksrcinfo && "
cmd+="git add . && git commit -m 'Update for kernel $(full_kernel_version ${AZB_KERNEL_VERSION})' && "
cmd+="git push"
run_cmd "${cmd}"
done
}
if [[ ${AZB_MODE_DEF} -eq 1 ]]; then
AZB_PKG_LIST=${AZB_DEF_PKG_LIST}
if [[ ${AZB_MODE_STD} -eq 1 ]]; then
AZB_KERNEL_VERSION=${AZB_STD_KERNEL_VERSION}
AZB_PKG_LIST=${AZB_STD_PKG_LIST}
push_packages
elif [[ ${AZB_MODE_GIT} -eq 1 ]]; then
AZB_KERNEL_VERSION=${AZB_GIT_KERNEL_VERSION}
AZB_PKG_LIST=${AZB_GIT_PKG_LIST}
push_packages
elif [[ ${AZB_MODE_LTS} -eq 1 ]]; then
AZB_KERNEL_VERSION=${AZB_LTS_KERNEL_VERSION}
AZB_PKG_LIST=${AZB_LTS_PKG_LIST}
push_packages
fi

@ -1,26 +1,44 @@
#!/bin/bash
#!/bin/bash -e
#
# repo.sh adds the archzfs packages to a specified repository.
# repo.sh adds the archzfs packages to the archzfs repository or archzfs-testing repository
#
source ./lib.sh
source ./conf.sh
set -e
NAME=$(basename $0)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
if ! source ${SCRIPT_DIR}/lib.sh; then
echo "!! ERROR !! -- Could not load lib.sh!"
fi
if ! source ${SCRIPT_DIR}/conf.sh; then
error "Could not load conf.sh!"
fi
trap 'trap_abort' INT QUIT TERM HUP
trap 'trap_exit' EXIT
DRY_RUN=0 # Show commands only. Don't do anything.
AZB_REPO="" # The destination repo for the packages
DRY_RUN=0 # Show commands only. Don't do anything.
AZB_REPO="" # The destination repo for the packages
AZB_KERNEL_VERSION=""
AZB_KERNEL_VERSION_NO_HYPHEN=""
AZB_PKGVER_MATCH=""
AZB_MODE=""
AZB_MODE_STD=0
AZB_MODE_GIT=0
AZB_MODE_LTS=0
usage() {
echo "repo.sh - Adds the compiled packages to the archzfs repo."
echo "${NAME} - Adds the compiled packages to the archzfs repo."
echo
echo "Usage: repo.sh [options] [mode] [repo] [package [...]]"
echo "Usage: ${NAME} [options] [mode] [repo] [package [...]]"
echo
echo "Options:"
echo
@ -30,102 +48,129 @@ usage() {
echo
echo "Modes:"
echo
echo " std Use the standard packages. Used by default."
echo " git Use the git packages."
echo " lts Use the lts packages."
echo
echo "Repository target:"
echo
echo " azfs Use the archzfs repo. Used by default."
echo " test Use the archzfs-testing repo."
echo
echo "Example Usage:"
echo
echo " repo.sh git core :: Add git packages in the current directory to the core repo."
echo " repo.sh lts core -n -d :: Show output commands and debug info."
echo " repo.sh git core package.tar.xz :: Add package.tar.xz to the core repo."
echo " repo.sh gts core *.tar.xz :: Add *.tar.xz to the core repo."
echo " ${NAME} :: Add standard packages to the archzfs repo."
echo " ${NAME} lts -n -d :: Show output commands and debug info."
echo " ${NAME} git package.tar.xz :: Add package.tar.xz to the archzfs repo."
echo " ${NAME} gts *.tar.xz :: Add *.tar.xz to the archzfs repo."
trap - EXIT # Prevents exit log output
}
ARGS=("$@")
for (( a = 0; a < $#; a++ )); do
if [[ ${ARGS[$a]} == "git" ]]; then
if [[ ${ARGS[$a]} == "std" ]]; then
AZB_MODE_STD=1
AZB_MODE="std"
elif [[ ${ARGS[$a]} == "git" ]]; then
AZB_MODE_STD=0
AZB_MODE_GIT=1
AZB_MODE="git"
elif [[ ${ARGS[$a]} == "lts" ]]; then
AZB_MODE_STD=0
AZB_MODE_LTS=1
elif [[ ${ARGS[$a]} == "core" ]]; then
AZB_REPO="demz-repo-core"
elif [[ ${ARGS[$a]} == "community" ]]; then
AZB_REPO="demz-repo-community"
elif [[ ${ARGS[$a]} == "testing" ]]; then
AZB_REPO="demz-repo-testing"
elif [[ ${ARGS[$a]} == "archiso" ]]; then
AZB_REPO="demz-repo-archiso"
elif [[ ${ARGS[$a]} == "-h" ]]; then
usage;
exit 0;
AZB_MODE="lts"
elif [[ ${ARGS[$a]} == "azfs" ]]; then
AZB_REPO="archzfs"
elif [[ ${ARGS[$a]} == "test" ]]; then
AZB_REPO="archzfs-testing"
elif [[ ${ARGS[$a]} == "-n" ]]; then
DRY_RUN=1
elif [[ ${ARGS[$a]} == "-d" ]]; then
DEBUG=1
elif [[ ${ARGS[$a]} == "-h" ]]; then
usage;
exit 0;
fi
done
if [ $# -lt 1 ]; then
if [[ $# -lt 1 ]]; then
usage;
exit 0;
fi
if [[ $AZB_MODE_GIT == 0 && $AZB_MODE_LTS == 0 ]]; then
echo -e "\n"
if [[ ${AZB_REPO} == "" ]]; then
error "No destination repo specified!"
exit 1
fi
if [[ ${AZB_MODE} == "" ]]; then
echo
error "A mode must be selected!"
echo -e "\n"
echo
usage;
exit 0;
fi
[[ $AZB_MODE_GIT == 1 ]] && AZB_KERNEL_VERSION=$AZB_GIT_KERNEL_VERSION || AZB_KERNEL_VERSION=$AZB_LTS_KERNEL_VERSION
msg "repo.sh started..."
msg "$(date) :: ${NAME} started..."
if [[ $AZB_REPO == "" ]]; then
error "No destination repo specified!"
exit 1
fi
# The abs path to the repo
AZB_REPO_TARGET=$AZB_REPO_BASEPATH/$AZB_REPO
AZB_REPO_TARGET=${AZB_REPO_BASEPATH}/${AZB_REPO}
# Set the kernel version
if [[ ${AZB_MODE_STD} -eq 1 ]]; then
AZB_KERNEL_VERSION=$(full_kernel_version ${AZB_STD_KERNEL_VERSION})
AZB_KERNEL_VERSION_NO_HYPHEN=$(full_kernel_version_no_hyphen ${AZB_STD_KERNEL_VERSION})
AZB_PKGVER_MATCH="${AZB_ZOL_VERSION}_${AZB_KERNEL_VERSION_NO_HYPHEN}-${AZB_STD_PKGREL}"
elif [[ ${AZB_MODE_GIT} -eq 1 ]]; then
AZB_KERNEL_VERSION=$(full_kernel_version ${AZB_GIT_KERNEL_VERSION})
AZB_KERNEL_VERSION_NO_HYPHEN=$(full_kernel_version_no_hyphen ${AZB_GIT_KERNEL_VERSION})
AZB_PKGVER_MATCH="${AZB_ZOL_VERSION}_${AZB_KERNEL_VERSION_NO_HYPHEN}-${AZB_GIT_PKGREL}"
elif [[ ${AZB_MODE_LTS} -eq 1 ]]; then
AZB_KERNEL_VERSION=$(full_kernel_version ${AZB_LTS_KERNEL_VERSION})
AZB_KERNEL_VERSION_NO_HYPHEN=$(full_kernel_version_no_hyphen ${AZB_LTS_KERNEL_VERSION})
AZB_PKGVER_MATCH="${AZB_ZOL_VERSION}_${AZB_KERNEL_VERSION_NO_HYPHEN}-${AZB_LTS_PKGREL}"
fi
debug "DRY_RUN: "${DRY_RUN}
debug "AZB_REPO: "${AZB_REPO}
debug "AZB_REPO_TARGET: ${AZB_REPO_TARGET}"
debug "AZB_KERNEL_VERSION: ${AZB_KERNEL_VERSION}"
debug "AZB_KERNEL_VERSION_NO_HYPHEN: ${AZB_KERNEL_VERSION_NO_HYPHEN}"
debug "AZB_PKGVER_MATCH: ${AZB_PKGVER_MATCH}"
debug "DRY_RUN: "$DRY_RUN
debug "AZB_REPO: "$AZB_REPO
debug "AZB_REPO_TARGET: $AZB_REPO_TARGET"
debug "AZB_KERNEL_VERSION: $AZB_KERNEL_VERSION"
# A list of packages to install. Pulled from the command line.
pkgs=()
# Extract any packages from the arguments passed to the script
for arg in "$@"; do
if [[ $arg =~ pkg.tar.xz$ ]]; then
pkgs+=("${pkgs[@]}" $arg)
if [[ ${arg} =~ pkg.tar.xz$ ]]; then
pkgs+=("${pkgs[@]}" ${arg})
fi
done
[[ $AZB_MODE_GIT == 1 ]] && path_glob="*-git" || path_glob="*-lts"
# Get the local packages if no packages were passed to the script
if [[ "${#pkgs[@]}" -eq 0 ]]; then
# Get packages from the backup directory if the repo is demz-repo-archiso
if [[ $AZB_REPO == "demz-repo-archiso" ]]; then
fcmd_out=$(find ${AZB_REPO_BASEPATH}/demz-repo-core -iname "*${AZB_KERNEL_ARCHISO_VERSION}*.pkg.tar.xz")
if [[ $fcmd_out == "" ]]; then
fcmd_out=$(find ${AZB_PACKAGE_BACKUP_DIR} -iname "*${AZB_KERNEL_ARCHISO_VERSION}*.pkg.tar.xz")
fi
else
fcmd_out=$(find ${path_glob} -iname "*${AZB_KERNEL_VERSION}*.pkg.tar.xz")
fi
for pkg in $fcmd_out; do
pkgs+=($pkg)
done
fi
if [[ $AZB_REPO != "" ]]; then
if [[ ${AZB_REPO} != "" ]]; then
msg "Creating a list of packages to add..."
# Get the local packages if no packages were passed to the script
if [[ "${#pkgs[@]}" -eq 0 ]]; then
# Get packages from the backup directory if the repo is demz-repo-archiso
run_cmd_show_and_capture_output_no_dry_run "find packages/${AZB_MODE}/ -iname '*${AZB_KERNEL_VERSION_NO_HYPHEN}*.pkg.tar.xz'"
for pkg in ${RUN_CMD_OUTPUT}; do
pkgs+=(${pkg})
done
fi
# A list of packages to add. The strings are in the form of
# "name;pkg.tar.xz;repo_path". There must be no spaces.
pkg_list=()
@ -133,52 +178,35 @@ if [[ $AZB_REPO != "" ]]; then
# A list of package sources to move
pkg_src_list=()
[[ $AZB_MODE_GIT == 1 ]] && full_kernel_git_version || full_kernel_lts_version
[[ $AZB_REPO == "demz-repo-archiso" ]] && full_kernel_archiso_version
for pkg in ${pkgs[@]}; do
arch=$(package_arch_from_path ${pkg})
name=$(package_name_from_path ${pkg})
vers=$(package_version_from_path ${pkg})
arch=$(package_arch_from_path $pkg)
name=$(package_name_from_path $pkg)
vers=$(package_version_from_path $pkg)
debug "Version match check: arch: ${arch} name: ${name} vers: ${vers} AZB_PKGVER_MATCH: ${AZB_PKGVER_MATCH}"
version_match=0
# Use a specific version incase of archiso
if [[ $AZB_REPO == "demz-repo-archiso" ]]; then
debug "Expect version: ${AZB_ZOL_VERSION}.*${AZB_KERNEL_ARCHISO_VERSION_CLEAN}-${AZB_ARCHISO_PKGREL}"
[[ $vers =~ ${AZB_ZOL_VERSION}.*${AZB_KERNEL_ARCHISO_VERSION_CLEAN}-${AZB_ARCHISO_PKGREL} ]] && version_match=1
elif [[ $AZB_REPO == "demz-repo-core" && $AZB_MODE_GIT == 1 ]]; then
[[ $vers =~ ${AZB_ZOL_VERSION}.*${AZB_GIT_KERNEL_X64_VERSION_CLEAN}-${AZB_GIT_PKGREL} ]] && version_match=1
elif [[ $AZB_REPO == "demz-repo-core" && $AZB_MODE_LTS == 1 ]]; then
[[ $vers =~ ${AZB_ZOL_VERSION}.*${AZB_LTS_KERNEL_X64_VERSION_CLEAN}-${AZB_LTS_PKGREL} ]] && version_match=1
fi
if [[ $version_match -eq 0 ]]; then
if [[ ${vers} != ${AZB_PKGVER_MATCH} ]]; then
debug "Version mismatch!"
continue
fi
if [[ $arch == "any" ]]; then
repos=`realpath $AZB_REPO_TARGET/{x86_64,i686}`
for repo in $repos; do
debug "Package: pkgname: $name\n\t\t pkgver: $vers\n\t\t pkgpath: $pkg\n\t\t pkgdest: $AZB_REPO_TARGET/$arch"
if [[ ${arch} == "any" ]]; then
repos=`realpath ${AZB_REPO_TARGET}/x86_64`
for repo in ${repos}; do
debug "Package: pkgname: ${name} pkgver: ${vers} pkgpath: ${pkg} pkgdest: ${AZB_REPO_TARGET}/${arch}"
# Each index is [name, version, pkgpath, pkgdest]
pkg_list+=("$name;$vers;$pkg;$repo")
pkg_list+=("${name};${vers};${pkg};${repo}")
done
continue
fi
debug "Using: pkgname: $name\n\t\t pkgver: $vers\n\t\t pkgpath: $pkg\n\t\t pkgdest: $AZB_REPO_TARGET/$arch"
pkg_list+=("$name;$vers;$pkg;$AZB_REPO_TARGET/$arch")
debug "Using: pkgname: ${name} pkgver: ${vers} pkgpath: ${pkg} pkgdest: ${AZB_REPO_TARGET}/${arch}"
pkg_list+=("${name};${vers};${pkg};${AZB_REPO_TARGET}/${arch}")
if [[ $AZB_REPO == "demz-repo-archiso" ]]; then
litem="$AZB_REPO_BASEPATH/demz-repo-core/$arch/$name-$vers.src.tar.gz;$AZB_REPO_TARGET/$arch"
else
litem="$name/$name-$vers.src.tar.gz;$AZB_REPO_TARGET/$arch"
fi
debug "Source: srcname: $name-$vers.src.tar.gz\n\t\t srcdest: $AZB_REPO_TARGET/$arch"
pkg_src_list+=($litem)
litem="packages/${AZB_MODE}/${name}/${name}-${vers}.src.tar.gz;${AZB_REPO_TARGET}/${arch}"
debug "Source: srcname: ${name}-${vers}.src.tar.gz srcdest: ${AZB_REPO_TARGET}/${arch}"
pkg_src_list+=(${litem})
done
if [[ ${#pkg_list[@]} == 0 ]]; then
@ -192,60 +220,52 @@ if [[ $AZB_REPO != "" ]]; then
src_mv_list=()
for ipkg in ${pkg_list[@]}; do
IFS=';' read -a pkgopt <<< "$ipkg"
IFS=';' read -a pkgopt <<< "${ipkg}"
name="${pkgopt[0]}"
vers="${pkgopt[1]}"
pkgp="${pkgopt[2]}"
repo="${pkgopt[3]}"
msg2 "Processing $name-$vers to $repo"
[[ ! -d $repo ]] && run_cmd "mkdir -p $repo"
msg2 "Processing package ${name}-${vers} to ${repo}"
[[ ! -d ${repo} ]] && run_cmd "mkdir -p ${repo}"
# Move the old packages to backup
for x in $(find $repo -type f -iname "${name}*.pkg.tar.xz"); do
ename=$(package_name_from_path $x)
evers=$(package_version_from_path $x)
if [[ $ename == $name && $evers != $vers ]]; then
for x in $(find ${repo} -type f -iname "${name}*.pkg.tar.xz"); do
ename=$(package_name_from_path ${x})}
evers=$(package_version_from_path ${x})}
if [[ ${ename} == ${name} && ${evers} != ${vers} ]]; then
# The '*' globs the signatures and package sources
epkg="$repo/$ename-${evers}*"
exist_pkg_mv_list+=($epkg)
epkg="${repo}/${ename}-${evers}*"
exist_pkg_mv_list+=(${epkg})
fi
done
# The * is to catch the signature
new_pkg_cp_list+=("$pkgp*;$repo")
bname=$(basename $pkgp)
pkg_add_list+=("$repo/$bname;$repo")
new_pkg_cp_list+=("${pkgp}*;${repo}")
bname=$(basename ${pkgp})
pkg_add_list+=("${repo}/${bname};${repo}")
done
# Build mv list with unique source packages since i686 and x86_64 both have
# identical source packages. If we attempt to move with identical file
# names, cp will fail with the "cp: will not overwrite just-created" error.
# Build mv list with unique source packages since i686 and x86_64 both have identical source packages. If we attempt to
# move with identical file names, cp will fail with the "cp: will not overwrite just-created" error.
exist_pkg_mv_list_uniq=()
for ((i = 0; i < ${#exist_pkg_mv_list[@]}; i++)); do
if [[ ${exist_pkg_mv_list[$i]} != *src.tar.gz ]]; then
exist_pkg_mv_list_uniq+=(${exist_pkg_mv_list[$i]})
continue
fi
if [[ ${exist_pkg_mv_list[$i]} == *x86_64* ]]; then
exist_pkg_mv_list_uniq+=(${exist_pkg_mv_list[$i]})
fi
done
msg "Performing file operations..."
if [[ ${#exist_pkg_mv_list_uniq[@]} -gt 0 && $AZB_REPO != "demz-repo-archiso" ]]; then
msg2 "Copy old packages and sources to backup directory"
run_cmd "cp -f ${exist_pkg_mv_list_uniq[*]} $AZB_PACKAGE_BACKUP_DIR/"
fi
# Remove the existing packages in the repo path
run_cmd "rm -f ${exist_pkg_mv_list[*]}"
for arch in "i686" "x86_64"; do
if [[ ${exist_pkg_mv_list[@]} -ne 0 ]]; then
run_cmd "rm -f ${exist_pkg_mv_list[*]}"
fi
msg "Copying the new $arch packages to the repo..."
for arch in "x86_64"; do
msg "Copying the new ${arch} packages to the repo..."
cp_list="" # The packages to copy in one string
ra_list="" # The packages to add to the repo in one string
@ -253,43 +273,38 @@ if [[ $AZB_REPO != "" ]]; then
# Create the command file lists from the arrays
for pkg in "${new_pkg_cp_list[@]}"; do
if [[ "$pkg" == *$arch* ]]; then
cp_list="$cp_list "$(echo "$pkg" | cut -d \; -f 1)
repo=$(echo "$pkg" | cut -d \; -f 2)
ra=$(echo "$pkg" | cut -d \; -f 1 | xargs basename)
ra_list="$ra_list $repo/${ra%?}"
if [[ "${pkg}" == *${arch}* ]]; then
cp_list="$cp_list "$(echo "${pkg}" | cut -d \; -f 1)
repo=$(echo "${pkg}" | cut -d \; -f 2)
ra=$(echo "${pkg}" | cut -d \; -f 1 | xargs basename)
ra_list="${ra_list} ${repo}/${ra%?}"
fi
done
if [[ $cp_list == "" ]]; then
if [[ ${cp_list} == "" ]]; then
warning "No packages to copy!"
continue
fi
run_cmd "cp -f $cp_list $repo/"
run_cmd "repo-add -k $AZB_GPG_SIGN_KEY -s -v $repo/${AZB_REPO}.db.tar.xz $ra_list"
if [[ $? -ne 0 ]]; then
run_cmd "cp -fv ${cp_list} ${repo}/"
run_cmd "repo-add -k ${AZB_GPG_SIGN_KEY} -s -v ${repo}/${AZB_REPO}.db.tar.xz ${ra_list}"
if [[ ${RUN_CMD_RETURN} -ne 0 ]]; then
error "An error occurred adding the package to the repo!"
exit 1
fi
done
# Copy package sources
msg "Copy package sources"
for arch in "i686" "x86_64"; do
for arch in "x86_64"; do
src_cp_list=()
for src in "${pkg_src_list[@]}"; do
if [[ "$src" == *$arch* ]]; then
src_cp_list="$src_cp_list "$(echo "$src" | cut -d \; -f 1)
repo=$(echo "$src" | cut -d \; -f 2)
if [[ "${src}" == *${arch}* ]]; then
debug "SRC='${src}'"
src_cp_list="${src_cp_list} "$(echo "${src}" | cut -d \; -f 1)
repo=$(echo "${src}" | cut -d \; -f 2)
fi
done
run_cmd "cp $src_cp_list $repo/"
if [[ $arch == "x86_64" && $AZB_REPO != "demz-repo-archiso" ]]; then
# Delete the package sources
run_cmd "rm $src_cp_list"
fi
run_cmd "cp -fv ${src_cp_list} ${repo}/"
done
fi

@ -58,6 +58,9 @@ for (( a = 0; a < $#; a++ )); do
done
msg "$(date) :: ${NAME} started..."
CHECK_WEBPAGE_RETVAL=0
@ -128,9 +131,6 @@ if [[ $(ping -w 1 -c 1 8.8.8.8 &> /dev/null; echo $?) != 0 ]]; then
fi
msg "${NAME} started..."
check_archiso() {
#
# Check archiso kernel version (this will change when the archiso is updated)
@ -148,7 +148,7 @@ check_linux_kernel() {
#
msg "Checking the online package database for x86_64 linux kernel version changes..."
check_webpage "https://www.archlinux.org/packages/core/x86_64/linux/" "(?<=<h2>linux )[\d\.-]+(?=</h2>)" \
"${AZB_DEF_KERNEL_VERSION}"
"${AZB_STD_KERNEL_VERSION}"
check_result "x86_64 linux kernel package" "linux x86_64"
}

@ -1,69 +1,107 @@
#!/bin/bash
DIR="$( cd "$( dirname "$0" )" && pwd )"
source $DIR/lib.sh
source $DIR/conf.sh
#
# Makes sure the local archzfs repo matches what is live on archzfs.com
#
NAME=$(basename $0)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
if ! source ${SCRIPT_DIR}/lib.sh; then
echo "!! ERROR !! -- Could not load lib.sh!"
fi
if ! source ${SCRIPT_DIR}/conf.sh; then
error "Could not load conf.sh!"
fi
trap 'trap_abort' INT QUIT TERM HUP
trap 'trap_exit' EXIT
usage() {
echo "verifier.sh - Compares repository hashes."
echo "${NAME} - Compares repository hashes."
echo
echo "Usage: ./verifier.sh [options]"
echo "Usage: ${NAME} [options]"
echo
echo "Options:"
echo
echo " -h: Show help information."
echo " -d: Show debug info."
echo
echo "Examples:"
echo "Examples:"
echo
echo " verifier.sh -d :: Show debug output."
echo " ${NAME} -d :: Show debug output."
trap - EXIT # Prevents exit log output
}
ARGS=("$@")
for (( a = 0; a < $#; a++ )); do
if [[ ${ARGS[$a]} == "-h" ]]; then
if [[ ${ARGS[$a]} == "-d" ]]; then
DEBUG=1
elif [[ ${ARGS[$a]} == "-h" ]]; then
usage;
exit 0;
elif [[ ${ARGS[$a]} == "-d" ]]; then
DEBUG=1
fi
done
compute_local_repo_hash() {
# $1: The repository to compute
# Sets LOCAL_REPO_HASH
msg2 "Computing local $1 repository hashes..."
LFILES=$(cd $AZB_REPO_BASEPATH; sha256sum $1/*/* | sort -r)
debug "Repository hash list:"
debug "$LFILES"
run_cmd "cd $AZB_REPO_BASEPATH; sha256sum $1/*/*"
if [[ ${RUN_CMD_RETURN} != 0 ]]; then
error "Could not run local hash!"
exit 1
fi
LFILES=$(echo ${RUN_CMD_OUTPUT} | sort -r)
LOCAL_REPO_HASH=$(echo "$LFILES" | sha256sum | cut -f 1 -d' ')
msg2 "Local hash: $LOCAL_REPO_HASH"
}
compute_remote_repo_hash() {
# $1: The repository to compute
# Sets REMOTE_REPO_HASH
msg2 "Computing remote $1 repository hashes..."
RFILES=$(ssh $AZB_REMOTE_LOGIN "cd webapps/default; sha256sum $1/*/* | sort -r")
debug "Repository hash list:"
debug "$RFILES"
run_cmd "ssh $AZB_REMOTE_LOGIN 'cd webapps/default; sha256sum $1/*/*'"
if [[ ${RUN_CMD_RETURN} != 0 ]]; then
error "Could not run remote hash!"
exit 1
fi
RFILES=$(echo ${RUN_CMD_OUTPUT} | sort -r)
REMOTE_REPO_HASH=$(echo "$RFILES" | sha256sum | cut -f 1 -d' ')
msg2 "Remote hash: $REMOTE_REPO_HASH"
}
msg "$(date) :: ${NAME} started..."
# Bail if no internet
# Please thank Comcast for this requirement...
if [[ $(ping -w 1 -c 1 8.8.8.8 &> /dev/null; echo $?) != 0 ]]; then
exit 0;
fi
msg "verifier.sh started..."
HAS_ERROR=0
for REPO in 'demz-repo-archiso' 'demz-repo-core'; do
for REPO in 'archzfs'; do
msg "Checking ${REPO}..."
# compare_repo $REPO
compute_local_repo_hash $REPO
@ -76,6 +114,7 @@ for REPO in 'demz-repo-archiso' 'demz-repo-core'; do
msg2 "$REPO is in sync"
done
if [[ $HAS_ERROR -eq 1 ]]; then
exit 1;
fi

Loading…
Cancel
Save