From 187ff884dbf064ef80ebd47a3f49ca985cb0bdf5 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 12 Jan 2020 20:52:40 +0100 Subject: [PATCH] add modules test The build of package 'nodeinfo' is implicitly tested by the modules test. --- .travis.yml | 19 ++++++- test/make-test.nix | 44 +++++++++++++++ test/run-tests.sh | 106 +++++++++++++++++++++++++++++++++++++ test/test-script.py | 104 ++++++++++++++++++++++++++++++++++++ test/test.nix | 52 ++++++++++++++++++ test/use-stable-pkgs.patch | 41 ++++++++++++++ 6 files changed, 364 insertions(+), 2 deletions(-) create mode 100644 test/make-test.nix create mode 100755 test/run-tests.sh create mode 100644 test/test-script.py create mode 100644 test/test.nix create mode 100644 test/use-stable-pkgs.patch diff --git a/.travis.yml b/.travis.yml index 3835300..4fc3a42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: bionic # needed for KVM language: shell install: @@ -18,7 +19,7 @@ env: # CACHIX_SIGNING_KEY - secure: "xXCFZ7g+k5YmCGm8R8l3bZElVmt+RD1KscG3kGr5w4HyyDPTzFetPo+sT8bUpysDU0u3HWhfVhHtpog2mhNhwVl3tQwKXea3dHKC1i6ypBg3gjDngmJRR5wo++ocYDpK8qPaU7m/jHQTNFnTA4CbmMcc05GcYx/1Ai/ZGkNwWFjdIcVeOUoiol33gykMOXIGDg2qlXudt33wP53FHbX8L4fxzodWfAuxKK4AoGprxy5eSnU7LCaXxxJmu4HwuV+Ux2U1NfE/E33cvhlUvTQCswVSZFG06mg8rwhMG1ozsDvlL2itZlu/BeUQH5y3XMMlnJIUXUazkRBibf1w/ebVjpOF+anqkqmq8tcbFEa7T+RJeVTIsvP+L8rE8fcmuZtdg9hNmgRnLmaeT0vVwD1L2UqW9HdRyujdoS0jPYuoc1W7f1JQWfAPhBPQ1SrtKyNNqcbVJ34aN7b+4vCzRpQL1JTbmjzQIWhkiKN1qMo1v/wbIydW8yka4hc4JOfdQLaAJEPI1eAC1MLotSAegMnwKWE1dzm66MuPSipksYjZrvsB28cV4aCVUffIuRhrSr1i2afRHwTpNbK9U4/576hah15ftUdR79Sfkcoi1ekSQTFGRvkRIPYtkKLYwFa3jVA41qz7+IIZCf4TsApy3XDdFx91cRub7yPq9BeZ83A+qYQ=" jobs: - - PKG=nodeinfo STABLE=1 + - TestModules=1 STABLE=1 - PKG=hwi STABLE=1 - PKG=lightning-charge STABLE=1 - PKG=lightning-charge STABLE=0 @@ -33,7 +34,21 @@ env: - PKG=liquid-swap STABLE=1 script: - printf '%s (%s)\n' "$NIX_PATH" "$VER" - - time nix-instantiate -A $PKG --add-root ./drv --indirect + - | + getBuildExpr() { + if [[ $TestModules ]]; then + if [[ ! -e /dev/kvm ]]; then + >&2 echo "No KVM available on VM Host." + exit 1 + fi + sudo chmod go+rw /dev/kvm + test/run-tests.sh exprForCI + else + echo "(import ./. {}).$PKG" + fi + } + - buildExpr=$(getBuildExpr) + - time nix-instantiate -E "$buildExpr" --add-root ./drv --indirect - outPath=$(nix-store --query ./drv) - | if nix path-info --store https://nix-bitcoin.cachix.org $outPath &>/dev/null; then diff --git a/test/make-test.nix b/test/make-test.nix new file mode 100644 index 0000000..018ab27 --- /dev/null +++ b/test/make-test.nix @@ -0,0 +1,44 @@ +testArgs: + +let + pkgs = import { config = {}; overlays = []; }; + + # Stable nixpkgs doesn't yet include the Python testing framework. + # Use unstable nixpkgs and patch it so that it uses stable nixpkgs for the VM + # machine configuration. + testingPkgs = let + # unstable as of 2020-01-09 + rev = "9beb0d1ac2ebd6063efbdc4d3631f8ce137bbf90"; + src = builtins.fetchTarball { + url = "https://github.com/nixos/nixpkgs-channels/archive/${rev}.tar.gz"; + sha256 = "1v95779di35qhrz70p2v27kmwm09h8pgh74i1wc72v0zp3bdrf50"; + }; + in + pkgs.runCommand "nixpkgs-testing" {} '' + cp -r ${src} $out + cd $out + chmod +w -R . + patch -p1 < ${./use-stable-pkgs.patch} + ''; + + test = (import "${testingPkgs}/nixos/tests/make-test-python.nix") testArgs; + + # Fix the black Python code formatter that's used in the test to allow the test + # script to have longer lines. The default width of 88 chars is too restrictive for + # our script. + fixedTest = { system ? builtins.currentSystem, ... }@args: + let + pkgs = (import testingPkgs { inherit system; config = {}; overlays = []; } ); + pkgsFixed = pkgs // { + python3Packages = pkgs.python3Packages // { + black = pkgs.writeScriptBin "black" '' + fileToCheck=''${@:$#} + [[ $fileToCheck = *test-script ]] && extraArgs='--line-length 100' + exec ${pkgs.python3Packages.black}/bin/black $extraArgs "$@" + ''; + }; + }; + in + test (args // { pkgs = pkgsFixed; }); +in + fixedTest diff --git a/test/run-tests.sh b/test/run-tests.sh new file mode 100755 index 0000000..69a969f --- /dev/null +++ b/test/run-tests.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Modules integration test runner. +# The test (./test.nix) uses the NixOS testing framework and is executed in a VM. +# +# Usage: +# Run test +# ./run-tests.sh +# +# Run test and save result to avoid garbage collection +# ./run-tests.sh build --out-link /tmp/nix-bitcoin-test +# +# Run interactive test debugging +# ./run-tests.sh debug +# +# This starts the testing VM and drops you into a Python REPL where you can +# manually execute the tests from ./test-script.py + +set -eo pipefail + +numCPUs=${numCPUs:-$(nproc)} +# Min. 800 MiB needed to avoid 'out of memory' errors +memoryMiB=${memoryMiB:-2048} + +scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) + +getPkgs() { + nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" $1 +} +export NIX_PATH=nixpkgs=$(getPkgs nixpkgs):nixpkgs-unstable=$(getPkgs nixpkgs-unstable) + +# Run the test. No temporary files are left on the host system. +run() { + # TMPDIR is also used by the test driver for VM tmp files + export TMPDIR=$(mktemp -d -p /tmp nix-bitcoin-test.XXXXXX) + trap "rm -rf $TMPDIR" EXIT + + nix-build --out-link $TMPDIR/driver "$scriptDir/test.nix" -A driver + + # Variable 'tests' contains the Python code that is executed by the driver on startup + if [[ $1 == --interactive ]]; then + echo "Running interactive testing environment" + tests=$( + echo 'is_interactive = True' + # The test script raises an error when 'is_interactive' is defined so + # that it just loads the initial helper functions and stops before + # executing the actual tests + echo 'try:' + echo ' exec(os.environ["testScript"])' + echo 'except:' + echo ' pass' + # Start VM + echo 'start_all()' + # Start REPL + echo 'import code' + echo 'code.interact(local=globals())' + ) + else + tests='exec(os.environ["testScript"])' + fi + + echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" + [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on' + env -i \ + NIX_PATH="$NIX_PATH" \ + TMPDIR="$TMPDIR" \ + tests="$tests" \ + QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ + QEMU_NET_OPTS="$QEMU_NET_OPTS" \ + $TMPDIR/driver/bin/nixos-test-driver +} + +debug() { + run --interactive +} + +# Run the test by building the test derivation +build() { + vmTestNixExpr | nix-build --no-out-link "$@" - +} + +# On continuous integration nodes there are few other processes running alongside the +# test, so use more memory here for maximum performance. +exprForCI() { + memoryMiB=3072 + memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo) + memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo) + # Round down to nearest multiple of 50 MiB for improved test build caching + ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50)) + ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB + >&2 echo "Host memory: total $((memTotalKiB / 1024)) MiB, available $memAvailableMiB MiB, VM $memoryMiB MiB" + vmTestNixExpr +} + +vmTestNixExpr() { + cat < ]; + security.allowUserNamespaces = true; # reenable disabled option + }; + + machine = { pkgs, lib, ... }: with lib; { + imports = [ + ../modules/nix-bitcoin.nix + ../modules/secrets/generate-secrets.nix + # using the hardened profile increases total test duration by ~50%, so disable it for now + # hardened + ]; + + services.nix-bitcoin.enable = true; + services.bitcoind.extraConfig = mkForce "connect=0"; + + services.clightning.enable = true; + services.spark-wallet.enable = true; + services.lightning-charge.enable = true; + services.nanopos.enable = true; + + services.lnd.enable = true; + systemd.services.lnd.wantedBy = mkForce []; + + services.electrs.enable = true; + + services.liquidd = { + enable = true; + listen = mkForce false; + extraConfig = "noconnect=1"; + }; + + services.nix-bitcoin-webindex.enable = true; + + services.hardware-wallets = { + trezor = true; + ledger = true; + }; + + # to test that unused secrets are made inaccessible by 'setup-secrets' + systemd.services.generate-secrets.postStart = '' + install -o nobody -g nogroup -m777 <(:) /secrets/dummy + ''; + }; + + testScript = builtins.readFile ./test-script.py; +} diff --git a/test/use-stable-pkgs.patch b/test/use-stable-pkgs.patch new file mode 100644 index 0000000..d89cf2d --- /dev/null +++ b/test/use-stable-pkgs.patch @@ -0,0 +1,41 @@ +--- a/nixos/lib/build-vms.nix ++++ b/nixos/lib/build-vms.nix +@@ -30,10 +30,10 @@ rec { + buildVM = + nodes: configurations: + +- import ./eval-config.nix { ++ import { + inherit system; + modules = configurations ++ extraConfigurations; +- baseModules = (import ../modules/module-list.nix) ++ ++ baseModules = (import ) ++ + [ ../modules/virtualisation/qemu-vm.nix + ../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs + { key = "no-manual"; documentation.nixos.enable = false; } + + +services.connman doesn't exist in stable nixpkgs +--- a/nixos/modules/virtualisation/qemu-vm.nix ++++ b/nixos/modules/virtualisation/qemu-vm.nix +@@ -620,7 +620,6 @@ in + + # Wireless won't work in the VM. + networking.wireless.enable = mkVMOverride false; +- services.connman.enable = mkVMOverride false; + + # Speed up booting by not waiting for ARP. + networking.dhcpcd.extraConfig = "noarp"; + +The test driver assumed coreutils to be in PATH. This fix will be ported to upstream. +--- a/nixos/lib/testing-python.nix ++++ b/nixos/lib/testing-python.nix +@@ -155,7 +155,7 @@ in rec { + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR + "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ +- --run "export testScript=\"\$(cat $out/test-script)\"" \ ++ --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \ + --set VLANS '${toString vlans}' + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms + wrapProgram $out/bin/nixos-run-vms \