From e68cb010baec6f7638b0b1adb490a017667fc82b Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 22 Oct 2022 19:37:58 +0200 Subject: [PATCH] tests: define tests via flake Advantages: - Pure test evaluations - The test framework can now be used by flakes that extend nix-bitcoin - Most features of `run-tests.sh` are now accessible via `nix build`/`nix run`. We keep `run-tests.sh` for advanced features like `scenarioOverridesFile` and adhoc scenarios. Other changes: - `run-tests.sh` now builds aggregate VM tests like `basic` or `buildable` by creating all VMs in a single evaluation. This speeds up the tests and eases debugging by separating the eval and build steps. - Use the new `nix` CLI which has improved build output logging by prefixing output lines with the origin drv name. --- .cirrus.yml | 2 +- examples/README.md | 23 +++ flake.lock | 23 +++ flake.nix | 20 ++- test/ci/build.sh | 8 +- test/clightning-replication.nix | 11 +- test/lib/make-container.sh | 17 +- test/lib/make-test.nix | 132 +++++++++------ test/lib/run-vm-without-tests.sh | 30 ++++ test/lib/run-vm.sh | 50 ++++++ test/run-tests.sh | 273 +++++++++++++------------------ test/tests.nix | 139 ++++++++++------ 12 files changed, 451 insertions(+), 277 deletions(-) create mode 100644 test/lib/run-vm-without-tests.sh create mode 100644 test/lib/run-vm.sh diff --git a/.cirrus.yml b/.cirrus.yml index 430078f..ccbe158 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,7 +30,7 @@ task: # This script is run as root build_script: - echo "sandbox = true" >> /etc/nix/nix.conf - - nix shell --inputs-from . nixpkgs#{bash,coreutils,gawk,cachix} -c ./test/ci/build.sh + - nix shell --inputs-from . nixpkgs#{bash,coreutils,cachix} -c ./test/ci/build.sh $scenario - name: flake build_script: diff --git a/examples/README.md b/examples/README.md index 64a54df..0ede270 100644 --- a/examples/README.md +++ b/examples/README.md @@ -55,6 +55,8 @@ The internal test suite is also useful for exploring features.\ The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on the host system. +`run-tests.sh` requires Nix >= 2.10. + ```bash git clone https://github.com/fort-nix/nix-bitcoin cd nix-bitcoin/test @@ -83,6 +85,27 @@ c systemctl status bitcoind ``` See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation. +#### Flakes +Tests can also be directly accessed via Flakes: +```bash +# Build test +nix build --no-link ..#tests.default + +# Run a node in a VM. No tests are executed. +nix run ..#tests.default.vm + +# Run a Python test shell inside a VM node +nix run ..#tests.default.run -- --debug + +# Run a node in a container. Requires extra-container, systemd and root privileges +nix run ..#tests.default.container +nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05 + +# Run a command in a container +nix run ..#tests.default.container -- --run c nodeinfo +nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05 +``` + ### Real-world example Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org to see the configuration of a nix-bitcoin node that's used in production. diff --git a/flake.lock b/flake.lock index 468640b..14b7d68 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,27 @@ { "nodes": { + "extra-container": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1666443795, + "owner": "erikarvstedt", + "repo": "extra-container", + "rev": "3b69ecfd363983cdee4db7f5d118b0ca099d23ed", + "type": "github" + }, + "original": { + "owner": "erikarvstedt", + "repo": "extra-container", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1659877975, @@ -49,6 +71,7 @@ }, "root": { "inputs": { + "extra-container": "extra-container", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "nixpkgs-unstable": "nixpkgs-unstable" diff --git a/flake.nix b/flake.nix index d11c46b..3be5d2a 100644 --- a/flake.nix +++ b/flake.nix @@ -8,9 +8,14 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; + extra-container = { + url = "github:erikarvstedt/extra-container"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; }; - outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils }: + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, ... }: let supportedSystems = [ "x86_64-linux" @@ -18,6 +23,8 @@ "aarch64-linux" "armv7l-linux" ]; + + test = import ./test/tests.nix nixpkgs.lib; in { lib = { mkNbPkgs = { @@ -27,6 +34,10 @@ }: import ./pkgs { inherit pkgs pkgsUnstable; }; + test = { + inherit (test) scenarios; + }; + inherit supportedSystems; }; @@ -93,7 +104,12 @@ # Allow accessing the whole nested `nbPkgs` attrset (including `modulesPkgs`) # via this flake. # `packages` is not allowed to contain nested pkgs attrsets. - legacyPackages = nbPkgs; + legacyPackages = + nbPkgs // + (test.pkgs self pkgs) // + { + extra-container = self.inputs.extra-container.packages.${system}.default; + }; apps = rec { default = vm; diff --git a/test/ci/build.sh b/test/ci/build.sh index af80ffe..90d6b4f 100755 --- a/test/ci/build.sh +++ b/test/ci/build.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash # This script can also be run locally for testing: -# scenario=default ./build.sh +# ./build.sh # # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system. set -euo pipefail +scenario=$1 + if [[ -v CIRRUS_CI ]]; then if [[ ! -e /dev/kvm ]]; then >&2 echo "No KVM available on VM host." @@ -16,5 +18,5 @@ if [[ -v CIRRUS_CI ]]; then chmod o+rw /dev/kvm fi -# shellcheck disable=SC2154 -"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario" +cd "${BASH_SOURCE[0]%/*}" +exec ./build-to-cachix.sh --expr "(builtins.getFlake (toString ../..)).legacyPackages.\${builtins.currentSystem}.tests.$scenario" diff --git a/test/clightning-replication.nix b/test/clightning-replication.nix index 06aad37..70d8102 100644 --- a/test/clightning-replication.nix +++ b/test/clightning-replication.nix @@ -1,12 +1,9 @@ # You can run this test via `run-tests.sh -s clightningReplication` -let - nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs; -in -import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }: +makeTestVM: pkgs: with pkgs.lib; let - keyDir = nixpkgs + "/nixos/tests/initrd-network-ssh"; + keyDir = pkgs.path + "/nixos/tests/initrd-network-ssh"; keys = { server = keyDir + "/ssh_host_ed25519_key"; client = keyDir + "/id_ed25519"; @@ -29,7 +26,7 @@ let }; }; in -{ +makeTestVM { name = "clightning-replication"; nodes = let nodes = { @@ -150,4 +147,4 @@ in # A gocryptfs has been created on the server server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf") ''; -}) +} diff --git a/test/lib/make-container.sh b/test/lib/make-container.sh index 491cddf..b500b59 100755 --- a/test/lib/make-container.sh +++ b/test/lib/make-container.sh @@ -53,7 +53,10 @@ set -euo pipefail -export containerName=nb-test +# These vars are set by ../run-tests.sh +: "${container:=}" +: "${scriptDir:=}" + containerCommand=shell while [[ $# -gt 0 ]]; do @@ -69,11 +72,10 @@ while [[ $# -gt 0 ]]; do done containerBin=$(type -P extra-container) || true -if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) ]]; then +if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.11*) ]]; then echo - echo "Building extra-container. Skip this step by adding extra-container 0.10 to PATH." - nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \ - -A pinned.extra-container >/dev/null + echo "Building extra-container. Skip this step by adding extra-container 0.11 to PATH." + nix build --out-link /tmp/extra-container "$scriptDir"/..#extra-container # When this script is run as root, e.g. when run in an extra-container shell, # chown the gcroot symlink to the regular (login) user so that the symlink can be # overwritten when this script is run without root. @@ -83,7 +85,4 @@ if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) export PATH="/tmp/extra-container/bin${PATH:+:}$PATH" fi -read -rd '' src < "$scenarioOverridesFile" + makeTmpDir + export scenarioOverridesFile=$tmpDir/scenario-overrides.nix + echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" scenario=tmp fi # Run the test. No temporary files are left on the host system. run() { - # TMPDIR is also used by the test driver for VM tmp files - TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) - export TMPDIR - runAtExit+="rm -rf ${TMPDIR};" - - nix-build --out-link "$TMPDIR/driver" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver - - # Variable 'tests' contains the Python code that is executed by the driver on startup - if [[ $1 == --interactive ]]; then - echo "Running interactive testing environment" - tests=$( - echo 'is_interactive = True' - echo 'exec(open(os.environ["testScript"]).read())' - # Start VM - echo 'if "machine" in vars(): machine.start()' - # Start REPL. - # Use `code.interact` for the REPL instead of the builtin test driver REPL - # because it supports low featured terminals like Emacs' shell-mode. - echo 'import code' - echo 'code.interact(local=globals())' - ) - else - tests='exec(open(os.environ["testScript"]).read())' - fi - - echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on' - cd "$TMPDIR" # The VM creates a VDE control socket in $PWD - env -i \ - NIX_PATH="$NIX_PATH" \ - TMPDIR="$TMPDIR" \ - USE_TMPDIR=1 \ - QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ - QEMU_NET_OPTS="$QEMU_NET_OPTS" \ - "$TMPDIR/driver/bin/nixos-test-driver" <(echo "$tests") + makeTmpDir + buildTestAttr .run --out-link "$tmpDir/run-vm" + NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/run-vm/bin/run-vm" "$@" } debug() { - run --interactive -} - -evalTest() { - nix-instantiate --eval -E "($(vmTestNixExpr)).outPath" -} - -instantiate() { - nix-instantiate -E "$(vmTestNixExpr)" "$@" + run --debug } container() { - export scriptDir scenario + local nixosContainer + if ! nixosContainer=$(type -p nixos-container) \ + || grep -q '"/etc/nixos-containers"' "$nixosContainer"; then + local attr=container + else + # NixOS with `system.stateVersion` <22.05 + local attr=containerLegacy + fi + echo "Building container" + makeTmpDir + export container=$tmpDir/container + buildTestAttr ".$attr" --out-link "$container" + export scriptDir "$scriptDir/lib/make-container.sh" "$@" } # Run a regular NixOS VM vm() { - TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) - export TMPDIR - runAtExit+="rm -rf $TMPDIR;" - - nix-build --out-link "$TMPDIR/vm" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vmWithoutTests" - - echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - [[ $NB_TEST_ENABLE_NETWORK ]] || export QEMU_NET_OPTS="restrict=on,$QEMU_NET_OPTS" - - # shellcheck disable=SC2211 - USE_TMPDIR=1 \ - NIX_DISK_IMAGE=$TMPDIR/img.qcow2 \ - QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS" \ - "$TMPDIR"/vm/bin/run-*-vm -} - -doBuild() { - name=$1 - shift - if [[ $ciBuild ]]; then - "$scriptDir/ci/build-to-cachix.sh" "$@" - else - if [[ $outLinkPrefix ]]; then - outLink="--out-link $outLinkPrefix-$name" - else - outLink=--no-out-link - fi - nix-build $outLink "$@" - fi + makeTmpDir + buildTestAttr .vm --out-link "$tmpDir/vm" + NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/vm/bin/run-vm-in-tmpdir" } # Run the test by building the test derivation buildTest() { - vmTestNixExpr | doBuild $scenario "$@" - + buildTestAttr "" "$@" } -vmTestNixExpr() { - extraQEMUOpts= - - if [[ $ciBuild ]]; then - # On continuous integration nodes there are few other processes running alongside the - # test, so use more memory here for maximum performance. - memoryMiB=4096 - memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo) - memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo) - # Round down to nearest multiple of 50 MiB for improved test build caching - # shellcheck disable=SC2017 - ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50)) - ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB - >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB" - >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB" - - # VMX is usually not available on CI nodes due to recursive virtualisation. - # Explicitly disable VMX, otherwise QEMU 4.20 fails with message - # "error: failed to set MSR 0x48b to 0x159ff00000000" - extraQEMUOpts="-cpu host,-vmx" - fi - - cat <&1) == *"requires a sub-command"* ]]; then - hasFlakes=1 - else - hasFlakes= - fi +buildTestAttr() { + local attr=$1 + shift + # TODO-EXTERNAL: + # Simplify and switch to pure build when `nix build` can build flake function outputs + nixInstantiateTest "$attr" + nixBuild "$scenario" "$drv" "$@" +} + +buildTests() { + local -n tests=$1 + shift + # TODO-EXTERNAL: + # Simplify and switch to pure build when `nix build` can instantiate flake function outputs + # shellcheck disable=SC2207 + drvs=($(nixInstantiate "pkgs.instantiateTests \"${tests[*]}\"")) + for i in "${!tests[@]}"; do + testName=${tests[$i]} + drv=${drvs[$i]} + echo + echo "Building test '$testName'" + nixBuild "$testName" "$drv" "$@" + done +} + +# Instantiate an attr of the test defined in global var `scenario` +nixInstantiateTest() { + local attr=$1 + shift + if [[ ${scenarioOverridesFile:-} ]]; then + local file="extraScenariosFile = \"$scenarioOverridesFile\";" + else + local file= fi - if [[ ! $hasFlakes ]]; then - echo "Skipping test '$testName'. Nix flake support is not enabled." - return 1 + nixInstantiate "(pkgs.getTest { name = \"$scenario\"; $file })$attr" "$@" >/dev/null +} + +drv= +# Sets global var `drv` to the gcroot link of the instantiated derivation +nixInstantiate() { + local expr=$1 + shift + makeTmpDir + drv="$tmpDir/drv" + nix-instantiate --add-root "$drv" -E " + let + pkgs = (builtins.getFlake \"git+file://$scriptDir/..\").legacyPackages.\${builtins.currentSystem}; + in + $expr + " "$@" +} + +nixBuild() { + local outLinkName=$1 + shift + args=(--print-out-paths -L) + if [[ $outLinkPrefix ]]; then + args+=(--out-link "$outLinkPrefix-$outLinkName") + else + args+=(--no-link) fi + nix build "${args[@]}" "$@" } flake() { - if ! checkFlakeSupport "flake"; then return; fi - nix flake check "$scriptDir/.." } # Test generating module documentation for search.nixos.org nixosSearch() { - if ! checkFlakeSupport "nixosSearch"; then return; fi - if [[ $outLinkPrefix ]]; then # Add gcroots for flake-info nix build "$scriptDir/nixos-search#flake-info" -o "$outLinkPrefix-flake-info" @@ -285,26 +244,29 @@ nixosSearch() { } # A basic subset of tests to keep the total runtime within -# manageable bounds (<4 min on desktop systems). +# manageable bounds. # These are also run on the CI server. -basic() { - scenario=default buildTest "$@" - scenario=netns buildTest "$@" - scenario=netnsRegtest buildTest "$@" -} +basic=( + default + netns + netnsRegtest +) +basic() { buildTests basic "$@"; } # All tests that only consist of building a nix derivation. -# Their output is cached in /nix/store. -buildable() { - basic "$@" - scenario=full buildTest "$@" - scenario=regtest buildTest "$@" - scenario=hardened buildTest "$@" - scenario=clightningReplication buildTest "$@" - scenario=lndPruned buildTest "$@" -} +# shellcheck disable=2034 +buildable=( + "${basic[@]}" + full + regtest + hardened + clightningReplication + lndPruned +) +buildable() { buildTests buildable "$@"; } examples() { + # shellcheck disable=SC2016 script=' set -e runExample() { echo; echo Running example $1; ./$1; } @@ -317,7 +279,6 @@ examples() { } shellcheck() { - if ! checkFlakeSupport "shellcheck"; then return; fi "$scriptDir/shellcheck.sh" } diff --git a/test/tests.nix b/test/tests.nix index 5a38e6b..b0f9c69 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -1,17 +1,9 @@ # Integration tests, can be run without internet access. +lib: let - nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs; -in - -{ extraScenarios ? { ... }: {} -, pkgs ? import nixpkgs { config = {}; overlays = []; } -}: -with pkgs.lib; -let - globalPkgs = pkgs; - - baseConfig = { pkgs, config, ... }: let + # Included in all scenarios + baseConfig = { config, pkgs, ... }: with lib; let cfg = config.services; inherit (config.nix-bitcoin.lib.test) mkIfTest; in { @@ -32,9 +24,6 @@ let }; config = mkMerge [{ - # Share the same pkgs instance among tests - nixpkgs.pkgs = mkDefault globalPkgs; - environment.systemPackages = mkMerge (with pkgs; [ # Needed to test macaroon creation (mkIfTest "btcpayserver" [ openssl xxd ]) @@ -176,8 +165,9 @@ let ]; }; - scenarios = { - base = baseConfig; # Included in all scenarios + scenarios = with lib; { + # Included in all scenarios by ./lib/make-test.nix + base = baseConfig; default = scenarios.secureNode; @@ -273,7 +263,7 @@ let environment.systemPackages = [ pkgs.fping ]; }; - regtestBase = { config, ... }: { + regtestBase = { config, pkgs, ... }: { tests.regtest = true; test.data.num_blocks = 100; @@ -323,9 +313,10 @@ let services.lnd.enable = true; services.bitcoind.prune = 1000; }; + }; - ## Examples / debug helper - + ## Example scenarios that showcase extra features + exampleScenarios = with lib; { # Run a selection of tests in scenario 'netns' selectedTests = { imports = [ scenarios.netns ]; @@ -342,40 +333,82 @@ let # See ./lib/test-lib.nix for a description test.container.exposeLocalhost = true; }; - - adhoc = { - # - # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios. - }; }; +in { + inherit scenarios; - overrides = builtins.getEnv "scenarioOverridesFile"; - extraScenarios' = (if (overrides != "") then import overrides else extraScenarios) { - inherit scenarios pkgs; - inherit (pkgs) lib; + pkgs = flake: pkgs: rec { + # A basic test using the nix-bitcoin test framework + makeTestBasic = import ./lib/make-test.nix flake pkgs makeTestVM; + + # Wraps `makeTest` in NixOS' testing-python.nix so that the drv includes the + # log output and the test driver + makeTestVM = import ./lib/make-test-vm.nix pkgs; + + # A test using the nix-bitcoin test framework, with some helpful defaults + makeTest = { name ? "nix-bitcoin-test", config }: + makeTestBasic { + inherit name; + config = { + imports = [ + scenarios.base + config + ]; + # Share the same pkgs instance among tests + nixpkgs.pkgs = pkgs.lib.mkDefault pkgs; + }; + }; + + # A test using the nix-bitcoin test framework, with defaults specific to nix-bitcoin + makeTestNixBitcoin = { name, config }: + makeTest { + name = "nix-bitcoin-${name}"; + config = { + imports = [ config ]; + test.shellcheckServices.sourcePrefix = toString ./..; + }; + }; + + makeTests = scenarios: let + mainTests = builtins.mapAttrs (name: config: + makeTestNixBitcoin { inherit name config; } + ) scenarios; + in + { + clightningReplication = import ./clightning-replication.nix makeTestVM pkgs; + } // mainTests; + + tests = makeTests scenarios; + + ## Helper for ./run-tests.sh + + getTest = { name, extraScenariosFile ? null }: + let + tests = makeTests (scenarios // ( + lib.optionalAttrs (extraScenariosFile != null) + (import extraScenariosFile { + inherit scenarios lib pkgs; + nix-bitcoin = flake; + }) + )); + in + tests.${name} or (makeTestNixBitcoin { + inherit name; + config = { + services.${name}.enable = true; + }; + }); + + instantiateTests = testNames: + let + testNames' = lib.splitString " " testNames; + in + map (name: + let + test = tests.${name}; + in + builtins.seq (builtins.trace "Evaluating test '${name}'" test.outPath) + test + ) testNames'; }; - allScenarios = scenarios // extraScenarios'; - - makeTest = name: config: - makeTest' name { - imports = [ - allScenarios.base - config - ]; - }; - makeTest' = import ./lib/make-test.nix pkgs; - - tests = builtins.mapAttrs makeTest allScenarios // { - clightningReplication.vm = import ./clightning-replication.nix { - inherit pkgs; - inherit (pkgs.stdenv) system; - }; - }; - - getTest = name: tests.${name} or (makeTest name { - services.${name}.enable = true; - }); -in - tests // { - inherit getTest; - } +}