From 5e6dfda31dec35399cd627cd0088781c1455fecb Mon Sep 17 00:00:00 2001 From: Jesus Alvarez Date: Sat, 17 Sep 2016 01:16:13 -0700 Subject: [PATCH] Update std packages for kernel 4.7.4-1 --- packages/linux/spl-linux | 2 +- packages/linux/spl-linux-git | 2 +- packages/linux/spl-utils-linux | 2 +- packages/linux/spl-utils-linux-git | 2 +- packages/linux/zfs-linux | 2 +- packages/linux/zfs-linux-git | 2 +- packages/linux/zfs-utils-linux | 2 +- packages/linux/zfs-utils-linux-git | 2 +- src/kernels/linux.sh | 4 +- testing/test.sh | 11 ++- .../README.rst | 52 ++++++++++++ .../archiso.sh | 12 +++ .../archzfs-qemu-git-test-00-bootfs/boot.sh | 6 ++ .../archzfs-qemu-git-test-00-bootfs/chroot.sh | 29 +++++++ .../archzfs-qemu-git-test-00-bootfs/conf.sh | 20 +++++ .../archzfs-qemu-git-test-00-bootfs/config.sh | 26 ++++++ .../archzfs-qemu-git-test-00-bootfs/fs.sh | 85 +++++++++++++++++++ .../archzfs-qemu-git-test-00-bootfs/hooks.sh | 12 +++ .../archzfs-qemu-git-test-00-bootfs/pacman.sh | 39 +++++++++ .../syslinux.cfg | 8 ++ .../archzfs-qemu-git-test-00-bootfs/vm.sh | 0 .../archzfs-qemu-lts-test-00-bootfs/conf.sh | 2 +- .../archzfs-qemu-std-test-00-bootfs/conf.sh | 2 +- .../archzfs-qemu-std-test-00-default/conf.sh | 2 +- 24 files changed, 311 insertions(+), 15 deletions(-) create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/README.rst create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/archiso.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/boot.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/chroot.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/conf.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/config.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/fs.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/hooks.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/pacman.sh create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/syslinux.cfg create mode 100644 testing/tests/archzfs-qemu-git-test-00-bootfs/vm.sh diff --git a/packages/linux/spl-linux b/packages/linux/spl-linux index eba0f54..1cce4b4 160000 --- a/packages/linux/spl-linux +++ b/packages/linux/spl-linux @@ -1 +1 @@ -Subproject commit eba0f547c1d129d827401929bb75b09c1aeecb72 +Subproject commit 1cce4b4a484d30cc122ebe1e34764aee0d100f08 diff --git a/packages/linux/spl-linux-git b/packages/linux/spl-linux-git index e41569d..7f47b0f 160000 --- a/packages/linux/spl-linux-git +++ b/packages/linux/spl-linux-git @@ -1 +1 @@ -Subproject commit e41569df2793f734111442654d952a45b317da79 +Subproject commit 7f47b0ffa04dc6887e9a5e373c9d9266dc432f4a diff --git a/packages/linux/spl-utils-linux b/packages/linux/spl-utils-linux index e73a917..7799319 160000 --- a/packages/linux/spl-utils-linux +++ b/packages/linux/spl-utils-linux @@ -1 +1 @@ -Subproject commit e73a9171e1d2b7e3f2b62f761f5dfc17349bfedd +Subproject commit 77993191d671b291e78f97333e1e62bd43b72cd7 diff --git a/packages/linux/spl-utils-linux-git b/packages/linux/spl-utils-linux-git index cb6d59f..e3bfc77 160000 --- a/packages/linux/spl-utils-linux-git +++ b/packages/linux/spl-utils-linux-git @@ -1 +1 @@ -Subproject commit cb6d59fb1ddec40f7621600084827105f8abfe6a +Subproject commit e3bfc77fdfe10a743e5c7c57f8586db16dfeaf61 diff --git a/packages/linux/zfs-linux b/packages/linux/zfs-linux index bf1b396..9dd804a 160000 --- a/packages/linux/zfs-linux +++ b/packages/linux/zfs-linux @@ -1 +1 @@ -Subproject commit bf1b396a48b2d71becc29d85cc8d051ca07bae4c +Subproject commit 9dd804a2e3e94933b28830807ccc0265ea20af49 diff --git a/packages/linux/zfs-linux-git b/packages/linux/zfs-linux-git index 0ad7394..74f7fbf 160000 --- a/packages/linux/zfs-linux-git +++ b/packages/linux/zfs-linux-git @@ -1 +1 @@ -Subproject commit 0ad7394670dce0386e2a3a629b7bcbc13e57dbf7 +Subproject commit 74f7fbfbc7eac92faec961ca136b345bea4db616 diff --git a/packages/linux/zfs-utils-linux b/packages/linux/zfs-utils-linux index 2ef2188..dc03e72 160000 --- a/packages/linux/zfs-utils-linux +++ b/packages/linux/zfs-utils-linux @@ -1 +1 @@ -Subproject commit 2ef2188b531dc392764eb52e24b12c910996700d +Subproject commit dc03e721968f89e3ca86bb7aeedc858950034a20 diff --git a/packages/linux/zfs-utils-linux-git b/packages/linux/zfs-utils-linux-git index ddc9b7e..a50b57c 160000 --- a/packages/linux/zfs-utils-linux-git +++ b/packages/linux/zfs-utils-linux-git @@ -1 +1 @@ -Subproject commit ddc9b7e0751b31103a6078212564db097b348b8f +Subproject commit a50b57ce6954c207d69a92a06e1db13672f6dc75 diff --git a/src/kernels/linux.sh b/src/kernels/linux.sh index c103a24..6d5f57c 100644 --- a/src/kernels/linux.sh +++ b/src/kernels/linux.sh @@ -3,8 +3,8 @@ mode_name="std" mode_desc="Select and use the packages for the default linux kernel" # Kernel versions for default ZFS packages -pkgrel="7" -kernel_version="4.7.2-1" +pkgrel="1" +kernel_version="4.7.4-1" # Kernel version for GIT packages pkgrel_git="${pkgrel}" diff --git a/testing/test.sh b/testing/test.sh index 08e1f82..98ba27d 100755 --- a/testing/test.sh +++ b/testing/test.sh @@ -31,7 +31,7 @@ archiso_build() { run_cmd_no_output "cat ${script_dir}/../archiso/work/iso/arch/pkglist.x86_64.txt 2> /dev/null | grep linux-lts | grep -oP '(?<=core/linux-lts-).*$'" if [[ ${run_cmd_return} -ne 0 ]]; then build_archiso=1 - elif [[ ! -f "${packer_work_dir}/archlinux*.iso" ]]; then + elif [[ ! -f "$(find ${packer_work_dir} -maxdepth 1 -name 'archlinux*.iso' -print -quit)" ]]; then msg2 "archzfs archiso does not exist!" build_archiso=1 else @@ -49,7 +49,7 @@ archiso_build() { fi # Delete the working directories since we are out-of-date - run_cmd_no_output "rm -rf ${script_dir}/archiso/{out,work} ${packer_work_dir}/*.iso" + run_cmd_no_output "rm -rf ${script_dir}/../archiso/out ${script_dir}/../archiso/work ${packer_work_dir}/*.iso" source_safe "${test_mode}/conf.sh" && source_safe "${test_mode}/archiso.sh" && test_build_archiso } @@ -162,6 +162,10 @@ if [[ "${test_mode}" != "" ]]; then msg "Building arch base image" + if [[ ! -d "${packer_work_dir}" ]]; then + run_cmd "mkdir -p ${packer_work_dir}" + fi + if [[ -d "${packer_work_dir}/output-qemu" ]]; then msg2 "Deleting '${packer_work_dir}/output-qemu' because it should not exist" run_cmd "rm -rf ${packer_work_dir}/output-qemu" @@ -172,6 +176,9 @@ if [[ "${test_mode}" != "" ]]; then run_cmd "mkdir ${packer_work_dir}" fi + # Clear out everything except packer_cache and the archiso + run_cmd "find ${packer_work_dir} -mindepth 1 ! -iname 'mirrorlist' ! -iname 'archlinux*.iso' ! -iname 'packer_cache' -exec rm -rf {} \;" + if [[ ! -f "${packer_work_dir}/mirrorlist" ]]; then msg2 "Generating pacman mirrorlist" run_cmd "/usr/bin/reflector -c US -l 5 -f 5 --sort rate 2>&1 > ${packer_work_dir}/mirrorlist" diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/README.rst b/testing/tests/archzfs-qemu-git-test-00-bootfs/README.rst new file mode 100644 index 0000000..b9a8159 --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/README.rst @@ -0,0 +1,52 @@ +========================================= +Test archzfs-qemu-lts-test-01-root-bootfs +========================================= + +Tests all the steps required for archzfs-linux-lts to be used as a boot filesystem. + +-------- +Overview +-------- + +Builds a custom archiso with the linux-lts kernel used by packer to create a Qemu base image. Syslinux is used as the boot +loader. + +--- +How +--- + +1. The archzfs-linux-lts packages are built for the linux-lts kernel and added to a package repository named "archzfs-testing". + +#. The archzfs-testing repo is shared over NFS. + +#. A custom archiso is built that boots into the linux-lts kernel. See `Archiso customization`_ + +#. The test files are compressed into a tar archive. + +#. Packer is used to build a qemu base image using the custom archiso. + +#. `setup.sh` is ran in the archiso to install arch on ZFS. + +#. After installation of Arch on ZFS, the VM is rebooted and packer finalizes the base image. + +#. The qemu base image created by packer is booted, if the boot is successful, the test is considered passed. + +--------------------- +Archiso customization +--------------------- + +At the time of putting this test together (2016.09.03), there was no stable ZFSonLinux release that supported kernel 4.7 and +the archiso release at the time shipped with kernel 4.7. In order to install Arch on ZFS for test, I needed an archiso with +the linux-lts kernel. Thus, the archzfs-archiso was born! + +The archiso is built by `test.sh` and used by packer to create a Qemu base image with ZFS as the root filesystem for testing. + +The archiso comes with a bunch of features that are not needed in test, so they have been stripped out or modified. This +includes: + +* boot straight into the linux-lts kernel to speed up the testing cycle. +* ZFS does not support arch-i686, so it was stripped. +* iPXE was not needed. + +The archiso sources are copied from `/usr/share/archiso/configs/releng` after installation of the "archiso" package. The +modifed code is contained in the `testing/archiso-linux-lts` directory of this project. diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/archiso.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/archiso.sh new file mode 100644 index 0000000..cba15ae --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/archiso.sh @@ -0,0 +1,12 @@ +# We need an archiso with the lts kernel used by default +test_build_archiso() { + msg "Building archiso" + cd ${test_root_dir}/../../../archiso/ &> /dev/null + if [[ -d ${packer_work_dir}/out ]] && [[ $(ls -1 | wc -l) -gt 0 ]]; then + run_cmd "rm -rf ${test_root_dir}/archiso/out/archlinux*" + fi + run_cmd "./build.sh -v" + msg2 "Coping archiso to packer_work_dir" + run_cmd "cp ${test_root_dir}/../../../archiso/out/archlinux* ${packer_work_dir} && rm -rf ${test_root_dir}/archiso/work" + cd - &> /dev/null +} diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/boot.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/boot.sh new file mode 100644 index 0000000..b2fa575 --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/boot.sh @@ -0,0 +1,6 @@ +test_bootloader_install() { + # Setup the boot loader + run_cmd "mkdir -p ${arch_target_dir}/boot/syslinux; cp -f /root/syslinux.cfg '${arch_target_dir}/boot/syslinux/syslinux.cfg'" + run_cmd "arch-chroot ${arch_target_dir} /usr/bin/syslinux-install_update -i -a -m" + run_cmd_check 1 +} diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/chroot.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/chroot.sh new file mode 100644 index 0000000..029fc5f --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/chroot.sh @@ -0,0 +1,29 @@ +test_chroot_setup() { + # $1 arch-chroot target dir + msg "Setting up arch install..." + export arch_target_dir="${test_target_dir}" + if [[ -n $1 ]]; then + arch_target_dir="${1}" + fi + msg2 "Setting base image pacman mirror" + run_cmd "/usr/bin/cp /etc/pacman.d/mirrorlist ${arch_target_dir}/etc/pacman.d/mirrorlist" + + msg2 "generating the filesystem table" + run_cmd "/usr/bin/genfstab -p ${arch_target_dir} >> '${arch_target_dir}/etc/fstab'" + + # http://comments.gmane.org/gmane.linux.arch.general/48739 + msg2 "Adding workaround for shutdown race condition" + run_cmd "/usr/bin/install --mode=0644 poweroff.timer '${arch_target_dir}/etc/systemd/system/poweroff.timer'" + + msg2 "Create config.sh" + run_cmd "/usr/bin/install --mode=0755 /dev/null '${arch_target_dir}/usr/bin/config.sh'" + + # Special filesystem configure script + source_safe /root/test-config.sh + + msg2 "Entering chroot and configuring system" + run_cmd "/usr/bin/arch-chroot ${arch_target_dir} /usr/bin/config.sh" + + msg2 "Deleting config.sh" + rm ${arch_target_dir}/usr/bin/config.sh +} diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/conf.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/conf.sh new file mode 100644 index 0000000..c2ed44a --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/conf.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +export test_root_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +debug "test_root_dir='${test_root_dir}'" + +export test_target_dir='/mnt' + +export test_archzfs_repo_name="archzfs-testing" + +# Additional packages to install in the archiso +export test_archiso_packages="archzfs-linux-lts" + +# Additional packages to install in the archiso +export test_chroot_packages="$( "${arch_target_dir}/usr/bin/config.sh" + echo '${fqdn}' > /etc/hostname + /usr/bin/ln -s /usr/share/zoneinfo/${timezone} /etc/localtime + echo 'KEYMAP=${keymap}' > /etc/vconsole.conf + /usr/bin/sed -i 's/#${language}/${language}/' /etc/locale.gen + /usr/bin/locale-gen + + /usr/bin/sed -i 's/filesystems/zfs filesystems/' /etc/mkinitcpio.conf + + /usr/bin/mkinitcpio -p linux + + /usr/bin/usermod --password ${password} root + + # https://wiki.archlinux.org/index.php/Network_Configuration#Device_names + /usr/bin/ln -s /dev/null /etc/udev/rules.d/80-net-setup-link.rules + /usr/bin/ln -s '/usr/lib/systemd/system/dhcpcd@.service' '/etc/systemd/system/multi-user.target.wants/dhcpcd@eth0.service' + + # Configure ssh + sed -e '/^#PermitRootLogin prohibit-password$/c PermitRootLogin yes' \ + -e '/^#UseDNS no$/c UseDNS no' \ + -i /etc/ssh/sshd_config + + /usr/bin/systemctl enable sshd.service +EOF diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/fs.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/fs.sh new file mode 100644 index 0000000..1bafe92 --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/fs.sh @@ -0,0 +1,85 @@ +test_fs_config_nfs() { + # $1 arch-chroot directory + # prefix="${test_target_dir}" + if [[ -n $1 ]]; then + prefix="${1}" + fi + + msg "Create NFS mount points" + run_cmd "/usr/bin/mkdir -p ${prefix}/repo" + + msg "Setting the package cache (nfs mount)" + run_cmd "mount -t nfs4 -o rsize=32768,wsize=32768,timeo=3 10.0.2.2:/var/cache/pacman/pkg ${prefix}/var/cache/pacman/pkg" + + msg "Mounting the AUR package repo" + run_cmd "mount -t nfs4 -o rsize=32768,wsize=32768,timeo=3 10.0.2.2:/mnt/data/pacman/repo ${prefix}/repo" +} + + +test_fs_config_root_preinstall() { + msg "Configuring root filesystem!" + + export disk='/dev/vda' + export root_partition="${disk}1" + + msg2 "Clearing partition table on ${disk}" + run_cmd "sgdisk --zap ${disk}" + + msg2 "Destroying magic strings and signatures on ${disk}" + run_cmd "dd if=/dev/zero of=${disk} bs=512 count=2048" + run_cmd "wipefs --all ${disk}" + + # See http://www.rodsbooks.com/gdisk/sgdisk-walkthrough.html + # http://www.rodsbooks.com/gdisk/sgdisk.htm + msg2 "Creating boot partition on ${disk}" + run_cmd "sgdisk --new=1:0:512M --typecode=1:8300 ${disk}" + + msg2 "Creating root partition on ${disk}" + run_cmd "sgdisk --new=2:0:0 --typecode=2:bf00 ${disk}" + + msg2 "The disk" + run_cmd "sgdisk -p ${disk}" + + msg2 "Creating root filesystem" + run_cmd "zpool create -m ${test_target_dir} -f zroot /dev/vda2" + run_cmd "zfs create -o mountpoint=none zroot/ROOT" + run_cmd "zfs create -o compression=lz4 -o mountpoint=${test_target_dir}/ROOT zroot/ROOT/default" + run_cmd "zfs create -o mountpoint=none zroot/data" + run_cmd "zfs create -o compression=lz4 -o mountpoint=${test_target_dir}/ROOT/home zroot/data/home" + run_cmd "zfs set mountpoint=legacy zroot/data/home" + + msg2 "Mounting /home" + run_cmd "mount -t zfs -o default,noatime zroot/data/home ${test_target_dir}/ROOT/home" + + msg2 "Create boot directory" + run_cmd "mkdir -p ${test_target_dir}/ROOT/boot" + + msg2 "Creating /boot filesystem (ext4)" + run_cmd "mkfs.ext4 -F -m 0 -q -L boot /dev/vda1" + + msg2 "Mounting boot filesystem" + run_cmd "mount -o noatime,errors=remount-ro /dev/vda1 ${test_target_dir}/ROOT/boot" + +} + +test_fs_config_root_postinstall() { + msg "Performing final filesystem operations" + + msg2 "Unmounting boot partition" + run_cmd "umount ${test_target_dir}/ROOT/boot" + + msg2 "Unmounting nfs partitions" + run_cmd "umount -a -t nfs4" + + msg2 "Unmounting home partition" + run_cmd "umount ${test_target_dir}/ROOT/home" + + msg2 "Setting flags and exporting ZFS root" + run_cmd "zfs umount -a" + run_cmd "zpool set bootfs=zroot/ROOT/default zroot" + run_cmd "zfs set mountpoint=none zroot" + run_cmd "zfs set mountpoint=/ zroot/ROOT/default" + run_cmd "zfs set mountpoint=/home zroot/data/home" + run_cmd "zfs set mountpoint=legacy zroot/data/home" + run_cmd "zpool export zroot" +} diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/hooks.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/hooks.sh new file mode 100644 index 0000000..8a85176 --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/hooks.sh @@ -0,0 +1,12 @@ +#!/bin/bash + + +test_setup_exit() { + msg "Installation complete!" + systemctl reboot +} + + +test_met_acceptance_criteria() { + return 1 +} diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/pacman.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/pacman.sh new file mode 100644 index 0000000..610f4b6 --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/pacman.sh @@ -0,0 +1,39 @@ +# Requires the pacman cache and pacman package repos be mounted via NFS +test_pacman_config() { + # $1 arch-chroot target directory + arch_target_dir="" + arch_packages="${test_archiso_packages}" + if [[ -n $1 ]]; then + arch_target_dir="${1}" + arch_chroot="/usr/bin/arch-chroot ${1}" + fi + + msg "Installing archzfs repo into chroot" + printf "\n%s\n%s\n" "[${test_archzfs_repo_name}]" "Server = file:///repo/\$repo/\$arch" >> ${arch_target_dir}/etc/pacman.conf + + msg2 "Setting up gnupg" + run_cmd "${arch_chroot} dirmngr < /dev/null" + + msg2 "Installing the signer key" + run_cmd "${arch_chroot} pacman-key -r 0EE7A126" + run_cmd_check 1 + + run_cmd "${arch_chroot} pacman-key --lsign-key 0EE7A126" + run_cmd_check 1 + + if [[ ! -n $1 ]]; then + msg2 "Installing test packages" + # Install the required packages in the image + run_cmd "${arch_chroot} pacman -Sy --noconfirm ${arch_packages}" + run_cmd_check 1 + msg2 "Loading zfs modules" + run_cmd "modprobe zfs" + fi +} + + +test_pacman_pacstrap() { + msg "bootstrapping the base installation" + run_cmd "/usr/bin/pacstrap -c '${test_target_dir}/ROOT' base base-devel ${test_chroot_packages}" + run_cmd_check 1 +} diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/syslinux.cfg b/testing/tests/archzfs-qemu-git-test-00-bootfs/syslinux.cfg new file mode 100644 index 0000000..a5f3edc --- /dev/null +++ b/testing/tests/archzfs-qemu-git-test-00-bootfs/syslinux.cfg @@ -0,0 +1,8 @@ +DEFAULT arch +TIMEOUT 0.1 + +LABEL arch + MENU LABEL Arch Linux + LINUX ../vmlinuz-linux + APPEND zfs=zroot/ROOT/default rw + INITRD ../initramfs-linux.img diff --git a/testing/tests/archzfs-qemu-git-test-00-bootfs/vm.sh b/testing/tests/archzfs-qemu-git-test-00-bootfs/vm.sh new file mode 100644 index 0000000..e69de29 diff --git a/testing/tests/archzfs-qemu-lts-test-00-bootfs/conf.sh b/testing/tests/archzfs-qemu-lts-test-00-bootfs/conf.sh index 35d0e06..6f297a8 100644 --- a/testing/tests/archzfs-qemu-lts-test-00-bootfs/conf.sh +++ b/testing/tests/archzfs-qemu-lts-test-00-bootfs/conf.sh @@ -11,7 +11,7 @@ export test_archzfs_repo_name="archzfs-testing" export test_archiso_packages="archzfs-linux-lts" # Additional packages to install in the archiso -export test_chroot_packages="$(