tests: define tests via flake

Advantages:
- Pure test evaluations
- The test framework can now be used by flakes that extend nix-bitcoin
- Most features of `run-tests.sh` are now accessible via `nix build`/`nix run`.
  We keep `run-tests.sh` for advanced features like `scenarioOverridesFile` and adhoc scenarios.

Other changes:
- `run-tests.sh` now builds aggregate VM tests like `basic` or
  `buildable` by creating all VMs in a single evaluation.
  This speeds up the tests and eases debugging by separating the eval and build steps.
- Use the new `nix` CLI which has improved build output logging
  by prefixing output lines with the origin drv name.

.cirrus.yml

@@ -30,7 +30,7 @@ task:
       # This script is run as root
       build_script:
         - echo "sandbox = true" >> /etc/nix/nix.conf
-        - nix shell --inputs-from . nixpkgs#{bash,coreutils,gawk,cachix} -c ./test/ci/build.sh
+        - nix shell --inputs-from . nixpkgs#{bash,coreutils,cachix} -c ./test/ci/build.sh $scenario
 
     - name: flake
       build_script:

examples/README.md

@@ -55,6 +55,8 @@ The internal test suite is also useful for exploring features.\
 The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on
 the host system.
 
+`run-tests.sh` requires Nix >= 2.10.
+
 ```bash
 git clone https://github.com/fort-nix/nix-bitcoin
 cd nix-bitcoin/test
@@ -83,6 +85,27 @@ c systemctl status bitcoind
 ```
 See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.
 
+#### Flakes
+Tests can also be directly accessed via Flakes:
+```bash
+# Build test
+nix build --no-link ..#tests.default
+
+# Run a node in a VM. No tests are executed.
+nix run ..#tests.default.vm
+
+# Run a Python test shell inside a VM node
+nix run ..#tests.default.run -- --debug
+
+# Run a node in a container. Requires extra-container, systemd and root privileges
+nix run ..#tests.default.container
+nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05
+
+# Run a command in a container
+nix run ..#tests.default.container -- --run c nodeinfo
+nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05
+```
+
 ### Real-world example
 Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org
 to see the configuration of a nix-bitcoin node that's used in production.

flake.lock

@@ -1,5 +1,27 @@
 {
   "nodes": {
+    "extra-container": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1666443795,
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "rev": "3b69ecfd363983cdee4db7f5d118b0ca099d23ed",
+        "type": "github"
+      },
+      "original": {
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -49,6 +71,7 @@
     },
     "root": {
       "inputs": {
+        "extra-container": "extra-container",
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
         "nixpkgs-unstable": "nixpkgs-unstable"

flake.nix

@@ -8,9 +8,14 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05";
     nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
+    extra-container = {
+      url = "github:erikarvstedt/extra-container";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
   };
 
-  outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils }:
+  outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, ... }:
     let
       supportedSystems = [
         "x86_64-linux"
@@ -18,6 +23,8 @@
         "aarch64-linux"
         "armv7l-linux"
       ];
+
+      test = import ./test/tests.nix nixpkgs.lib;
     in {
       lib = {
         mkNbPkgs = {
@@ -27,6 +34,10 @@
         }:
           import ./pkgs { inherit pkgs pkgsUnstable; };
 
+        test = {
+          inherit (test) scenarios;
+        };
+
         inherit supportedSystems;
       };
 
@@ -93,7 +104,12 @@
         # Allow accessing the whole nested `nbPkgs` attrset (including `modulesPkgs`)
         # via this flake.
         # `packages` is not allowed to contain nested pkgs attrsets.
-        legacyPackages = nbPkgs;
+        legacyPackages =
+          nbPkgs //
+          (test.pkgs self pkgs) //
+          {
+            extra-container = self.inputs.extra-container.packages.${system}.default;
+          };
 
         apps = rec {
           default = vm;

test/ci/build.sh

@@ -1,12 +1,14 @@
 #!/usr/bin/env bash
 
 # This script can also be run locally for testing:
-#   scenario=default ./build.sh
+#   ./build.sh <scenario>
 #
 # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system.
 
 set -euo pipefail
 
+scenario=$1
+
 if [[ -v CIRRUS_CI ]]; then
     if [[ ! -e /dev/kvm ]]; then
         >&2 echo "No KVM available on VM host."
@@ -16,5 +18,5 @@ if [[ -v CIRRUS_CI ]]; then
     chmod o+rw /dev/kvm
 fi
 
-# shellcheck disable=SC2154
+cd "${BASH_SOURCE[0]%/*}"
-"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario"
+exec ./build-to-cachix.sh --expr "(builtins.getFlake (toString ../..)).legacyPackages.\${builtins.currentSystem}.tests.$scenario"

test/clightning-replication.nix

@@ -1,12 +1,9 @@
 # You can run this test via `run-tests.sh -s clightningReplication`
 
-let
+makeTestVM: pkgs:
-  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
-in
-import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }:
 with pkgs.lib;
 let
-  keyDir = nixpkgs + "/nixos/tests/initrd-network-ssh";
+  keyDir = pkgs.path + "/nixos/tests/initrd-network-ssh";
   keys = {
     server = keyDir + "/ssh_host_ed25519_key";
     client = keyDir + "/id_ed25519";
@@ -29,7 +26,7 @@ let
     };
   };
 in
-{
+makeTestVM {
   name = "clightning-replication";
 
   nodes = let nodes = {
@@ -150,4 +147,4 @@ in
           # A gocryptfs has been created on the server
           server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf")
   '';
-})
+}

test/lib/make-container.sh

@@ -53,7 +53,10 @@
 
 set -euo pipefail
 
-export containerName=nb-test
+# These vars are set by ../run-tests.sh
+: "${container:=}"
+: "${scriptDir:=}"
+
 containerCommand=shell
 
 while [[ $# -gt 0 ]]; do
@@ -69,11 +72,10 @@ while [[ $# -gt 0 ]]; do
 done
 
 containerBin=$(type -P extra-container) || true
-if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*) ]]; then
+if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.11*) ]]; then
     echo
-    echo "Building extra-container. Skip this step by adding extra-container 0.10 to PATH."
+    echo "Building extra-container. Skip this step by adding extra-container 0.11 to PATH."
-    nix-build --out-link /tmp/extra-container "$scriptDir"/../pkgs \
+    nix build --out-link /tmp/extra-container "$scriptDir"/..#extra-container
-      -A pinned.extra-container >/dev/null
     # When this script is run as root, e.g. when run in an extra-container shell,
     # chown the gcroot symlink to the regular (login) user so that the symlink can be
     # overwritten when this script is run without root.
@@ -83,7 +85,4 @@ if [[ ! ($containerBin && $(realpath "$containerBin") == *extra-container-0.10*)
     export PATH="/tmp/extra-container/bin${PATH:+:}$PATH"
 fi
 
-read -rd '' src <<EOF || true
+exec "$container"/bin/container "$containerCommand" "$@"
-((import "$scriptDir/tests.nix" {}).getTest "$scenario").container
-EOF
-exec extra-container "$containerCommand" -E "$src" "$@"

test/lib/make-test.nix

@@ -1,26 +1,23 @@
-pkgs:
+flake: pkgs: makeTestVM:
 let
-  makeVM = import ./make-test-vm.nix pkgs;
+  inherit (flake.inputs) extra-container;
-  inherit (pkgs) lib;
+  inherit (pkgs.stdenv.hostPlatform) system;
 in
 
-name: testConfig:
+{ name ? "nix-bitcoin-test", config }:
-{
+let
-  vm = makeVM {
+  inherit (pkgs) lib;
-    name = "nix-bitcoin-${name}";
+
+  testConfig = config;
+
+  test = makeTestVM {
+    inherit name;
 
     nodes.machine = { config, ... }: {
-      imports = [ testConfig ];
+      imports = [
-
+        testConfig
-      virtualisation = {
+        commonVmConfig
-        # Needed because duplicity requires 270 MB of free temp space, regardless of backup size
+      ];
-        diskSize = 1024;
-
-        # Min. 800 MiB needed to avoid 'out of memory' errors
-        memorySize = lib.mkDefault 2048;
-
-        cores = lib.mkDefault 2;
-      };
 
       test.shellcheckServices.enable = true;
     };
@@ -47,7 +44,7 @@ name: testConfig:
         (builtins.readFile ./../tests.py)
         cfg.test.extraTestScript
         # Don't run tests in interactive mode.
-        # is_interactive is set in ../run-tests.sh
+        # is_interactive is set in ./run-vm.sh
         ''
           if not "is_interactive" in vars():
               run_tests()
@@ -55,7 +52,15 @@ name: testConfig:
       ];
   };
 
-  container = {
+  # A VM runner for interactive use
+  run = pkgs.writers.writeBashBin "run-vm" ''
+    . ${./run-vm.sh} ${test.driver} "$@"
+  '';
+
+  mkContainer = legacyInstallDirs:
+    extra-container.lib.buildContainers {
+      inherit system legacyInstallDirs;
+      config = {
         # The container name has a 11 char length limit
         containers.nb-test = { config, ... }: {
           imports = [
@@ -83,12 +88,17 @@ name: testConfig:
           ];
         };
       };
+    };
+
+  container = mkContainer false;
+  containerLegacy = mkContainer true;
 
   # This allows running a test scenario in a regular NixOS VM.
   # No tests are executed.
-  vmWithoutTests = (pkgs.nixos ({ config, ... }: {
+  vm = (pkgs.nixos ({ config, ... }: {
     imports = [
       testConfig
+      commonVmConfig
       (pkgs.path + "/nixos/modules/virtualisation/qemu-vm.nix")
     ];
     virtualisation.graphics = false;
@@ -103,7 +113,37 @@ name: testConfig:
     '';
 
     system.stateVersion = lib.mkDefault config.system.nixos.release;
-  })).config.system.build.vm;
+  })).config.system.build.vm.overrideAttrs (old: {
+    meta = old.meta // { mainProgram = "run-vm-in-tmpdir"; };
+    buildCommand =  old.buildCommand + "\n" + ''
+      install -m 700 ${./run-vm-without-tests.sh} $out/bin/run-vm-in-tmpdir
+      patchShebangs $out/bin/run-vm-in-tmpdir
+    '';
+  });
+
+  commonVmConfig = {
+    virtualisation = {
+      # Needed because duplicity requires 270 MB of free temp space, regardless of backup size
+      diskSize = 1024;
+
+      # Min. 800 MiB needed to avoid 'out of memory' errors
+      memorySize = lib.mkDefault 2048;
+
+      # There are no perf gains beyond 3 cores.
+      # Benchmark: Ryzen 7 2700 (8 cores), VM test `default` as of 34f6eb90.
+      # Num. Cores    | 1   | 2  | 3  | 4  | 6
+      # Runtime (sec) | 125 | 95 | 89 | 89 | 90
+      cores = lib.mkDefault 3;
+    };
+  };
+in
+test // {
+  inherit
+    run
+    vm
+    container
+    # For NixOS with `system.stateVersion` <22.05
+    containerLegacy;
 
   config = testConfig;
 }

test/lib/run-vm-without-tests.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script uses the following env vars:
+# NIX_BITCOIN_VM_ENABLE_NETWORK
+# NIX_BITCOIN_VM_DATADIR
+# QEMU_OPTS
+# QEMU_NET_OPTS
+
+if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then
+    dataDir=$NIX_BITCOIN_VM_DATADIR
+else
+    dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+    trap 'rm -rf "$dataDir"' EXIT
+fi
+
+if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then
+    QEMU_NET_OPTS='restrict=on'
+fi
+
+# TODO-EXTERNAL:
+# Pass PATH because run-*-vm is impure (requires coreutils from PATH)
+env -i \
+    PATH="$PATH" \
+    USE_TMPDIR=1 \
+    TMPDIR="$dataDir" \
+    NIX_DISK_IMAGE="$dataDir/img.qcow2" \
+    QEMU_OPTS="${QEMU_OPTS:-}" \
+    QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \
+    "${BASH_SOURCE[0]%/*}"/run-*-vm

test/lib/run-vm.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script uses the following env vars:
+# NIX_BITCOIN_VM_ENABLE_NETWORK
+# NIX_BITCOIN_VM_DATADIR
+# QEMU_OPTS
+# QEMU_NET_OPTS
+
+if [[ ${NIX_BITCOIN_VM_DATADIR:-} ]]; then
+    dataDir=$NIX_BITCOIN_VM_DATADIR
+else
+    dataDir=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+    trap 'rm -rf "$dataDir"' EXIT
+fi
+
+testDriver=$1
+shift
+
+# Variable 'tests' contains the Python code that is executed by the driver on startup
+if [[ ${1:-} == --debug ]]; then
+    shift
+    echo "Running interactive testing environment"
+    # Start REPL.
+    # Use `code.interact` for the REPL instead of the builtin test driver REPL
+    # because it supports low featured terminals like Emacs' shell-mode.
+    tests='
+is_interactive = True
+exec(open(os.environ["testScript"]).read())
+if "machine" in vars(): machine.start()
+import code
+code.interact(local=globals())
+'
+    echo
+    echo "Starting VM, data dir: $dataDir"
+else
+    tests='exec(open(os.environ["testScript"]).read())'
+fi
+
+if [[ ! ${NIX_BITCOIN_VM_ENABLE_NETWORK:-} ]]; then
+    QEMU_NET_OPTS='restrict=on'
+fi
+
+# The VM creates a VDE control socket in $PWD
+env --chdir "$dataDir" -i \
+    USE_TMPDIR=1 \
+    TMPDIR="$dataDir" \
+    QEMU_OPTS="-nographic ${QEMU_OPTS:-}"  \
+    QEMU_NET_OPTS="${QEMU_NET_OPTS:-}" \
+    "$testDriver/bin/nixos-test-driver" <(echo "$tests") "$@"

test/run-tests.sh

@@ -66,7 +66,6 @@ scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
 args=("$@")
 scenario=
 outLinkPrefix=
-ciBuild=
 while :; do
     case $1 in
         --scenario|-s)
@@ -89,10 +88,6 @@ while :; do
                 exit 1
             fi
             ;;
-        --ci)
-            shift
-            ciBuild=1
-            ;;
         --copy-src|-c)
             shift
             if [[ ! $_nixBitcoinInCopiedSrc ]]; then
@@ -105,178 +100,142 @@ while :; do
     esac
 done
 
-numCPUs=${numCPUs:-$(nproc)}
+tmpDir=
-# Min. 800 MiB needed to avoid 'out of memory' errors
+# Sets global var `tmpDir`
-memoryMiB=${memoryMiB:-2048}
+makeTmpDir() {
-
+    if [[ ! $tmpDir ]]; then
-NIX_PATH=nixpkgs=$(nix eval --raw -f "$scriptDir/../pkgs/nixpkgs-pinned.nix" nixpkgs):nix-bitcoin=$(realpath "$scriptDir/..")
+        tmpDir=$(mktemp -d /tmp/nix-bitcoin-tests.XXX)
-export NIX_PATH
+        # shellcheck disable=SC2064
-
+        trap "rm -rf '$tmpDir'" EXIT
-runAtExit=
+    fi
-trap 'eval "$runAtExit"' EXIT
+}
 
 # Support explicit scenario definitions
 if [[ $scenario = *' '* ]]; then
-    scenarioOverridesFile=$(mktemp "${XDG_RUNTIME_DIR:-/tmp}/nb-scenario.XXX")
+    makeTmpDir
-    export scenarioOverridesFile
+    export scenarioOverridesFile=$tmpDir/scenario-overrides.nix
-
+    echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
-    # shellcheck disable=SC2016
-    runAtExit+='rm -f "$scenarioOverridesFile";'
-    echo "{ scenarios, pkgs, lib }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
     scenario=tmp
 fi
 
 # Run the test. No temporary files are left on the host system.
 run() {
-    # TMPDIR is also used by the test driver for VM tmp files
+    makeTmpDir
-    TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
+    buildTestAttr .run --out-link "$tmpDir/run-vm"
-    export TMPDIR
+    NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/run-vm/bin/run-vm" "$@"
-    runAtExit+="rm -rf ${TMPDIR};"
-
-    nix-build --out-link "$TMPDIR/driver" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vm" -A driver
-
-    # Variable 'tests' contains the Python code that is executed by the driver on startup
-    if [[ $1 == --interactive ]]; then
-        echo "Running interactive testing environment"
-        tests=$(
-            echo 'is_interactive = True'
-            echo 'exec(open(os.environ["testScript"]).read())'
-            # Start VM
-            echo 'if "machine" in vars(): machine.start()'
-            # Start REPL.
-            # Use `code.interact` for the REPL instead of the builtin test driver REPL
-            # because it supports low featured terminals like Emacs' shell-mode.
-            echo 'import code'
-            echo 'code.interact(local=globals())'
-        )
-    else
-        tests='exec(open(os.environ["testScript"]).read())'
-    fi
-
-    echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-    [[ $NB_TEST_ENABLE_NETWORK ]] || QEMU_NET_OPTS='restrict=on'
-    cd "$TMPDIR" # The VM creates a VDE control socket in $PWD
-    env -i \
-        NIX_PATH="$NIX_PATH" \
-        TMPDIR="$TMPDIR" \
-        USE_TMPDIR=1 \
-        QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS"  \
-        QEMU_NET_OPTS="$QEMU_NET_OPTS" \
-        "$TMPDIR/driver/bin/nixos-test-driver" <(echo "$tests")
 }
 
 debug() {
-    run --interactive
+    run --debug
-}
-
-evalTest() {
-    nix-instantiate --eval -E "($(vmTestNixExpr)).outPath"
-}
-
-instantiate() {
-    nix-instantiate -E "$(vmTestNixExpr)" "$@"
 }
 
 container() {
-    export scriptDir scenario
+    local nixosContainer
+    if ! nixosContainer=$(type -p nixos-container) \
+       || grep -q '"/etc/nixos-containers"' "$nixosContainer"; then
+        local attr=container
+    else
+        # NixOS with `system.stateVersion` <22.05
+        local attr=containerLegacy
+    fi
+    echo "Building container"
+    makeTmpDir
+    export container=$tmpDir/container
+    buildTestAttr ".$attr" --out-link "$container"
+    export scriptDir
     "$scriptDir/lib/make-container.sh" "$@"
 }
 
 # Run a regular NixOS VM
 vm() {
-    TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX)
+    makeTmpDir
-    export TMPDIR
+    buildTestAttr .vm --out-link "$tmpDir/vm"
-    runAtExit+="rm -rf $TMPDIR;"
+    NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/vm/bin/run-vm-in-tmpdir"
-
-    nix-build --out-link "$TMPDIR/vm" -E "((import \"$scriptDir/tests.nix\" {}).getTest \"$scenario\").vmWithoutTests"
-
-    echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-    [[ $NB_TEST_ENABLE_NETWORK ]] || export QEMU_NET_OPTS="restrict=on,$QEMU_NET_OPTS"
-
-    # shellcheck disable=SC2211
-    USE_TMPDIR=1 \
-    NIX_DISK_IMAGE=$TMPDIR/img.qcow2 \
-    QEMU_OPTS="-smp $numCPUs -m $memoryMiB -nographic $QEMU_OPTS"  \
-      "$TMPDIR"/vm/bin/run-*-vm
-}
-
-doBuild() {
-    name=$1
-    shift
-    if [[ $ciBuild ]]; then
-        "$scriptDir/ci/build-to-cachix.sh" "$@"
-    else
-        if [[ $outLinkPrefix ]]; then
-            outLink="--out-link $outLinkPrefix-$name"
-        else
-            outLink=--no-out-link
-        fi
-        nix-build $outLink "$@"
-    fi
 }
 
 # Run the test by building the test derivation
 buildTest() {
-    vmTestNixExpr | doBuild $scenario "$@" -
+    buildTestAttr "" "$@"
 }
 
-vmTestNixExpr() {
+evalTest() {
-    extraQEMUOpts=
+    nixInstantiateTest "" "$@"
-
+    # Print out path
-    if [[ $ciBuild ]]; then
+    nix-store -q "$drv"
-        # On continuous integration nodes there are few other processes running alongside the
+    # Print drv path
-        # test, so use more memory here for maximum performance.
+    realpath "$drv"
-        memoryMiB=4096
-        memTotalKiB=$(awk '/MemTotal/ { print $2 }' /proc/meminfo)
-        memAvailableKiB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
-        # Round down to nearest multiple of 50 MiB for improved test build caching
-        # shellcheck disable=SC2017
-        ((memAvailableMiB = memAvailableKiB / (1024 * 50) * 50))
-        ((memAvailableMiB < memoryMiB)) && memoryMiB=$memAvailableMiB
-        >&2 echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-        >&2 echo "Host memory total: $((memTotalKiB / 1024)) MiB, available: $memAvailableMiB MiB"
-
-        # VMX is usually not available on CI nodes due to recursive virtualisation.
-        # Explicitly disable VMX, otherwise QEMU 4.20 fails with message
-        # "error: failed to set MSR 0x48b to 0x159ff00000000"
-        extraQEMUOpts="-cpu host,-vmx"
-    fi
-
-    cat <<EOF
-    ((import "$scriptDir/tests.nix" {}).getTest "$scenario").vm.overrideAttrs (old: rec {
-      buildCommand = ''
-        export QEMU_OPTS="-smp $numCPUs -m $memoryMiB $extraQEMUOpts"
-        echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"
-      '' + old.buildCommand;
-    })
-EOF
 }
 
-checkFlakeSupport() {
+buildTestAttr() {
-    testName=$1
+    local attr=$1
-    if [[ ! -v hasFlakes ]]; then
+    shift
-        if [[ $(nix flake 2>&1) == *"requires a sub-command"* ]]; then
+    # TODO-EXTERNAL:
-            hasFlakes=1
+    # Simplify and switch to pure build when `nix build` can build flake function outputs
+    nixInstantiateTest "$attr"
+    nixBuild "$scenario" "$drv" "$@"
+}
+
+buildTests() {
+    local -n tests=$1
+    shift
+    # TODO-EXTERNAL:
+    # Simplify and switch to pure build when `nix build` can instantiate flake function outputs
+    # shellcheck disable=SC2207
+    drvs=($(nixInstantiate "pkgs.instantiateTests \"${tests[*]}\""))
+    for i in "${!tests[@]}"; do
+        testName=${tests[$i]}
+        drv=${drvs[$i]}
+        echo
+        echo "Building test '$testName'"
+        nixBuild "$testName" "$drv" "$@"
+    done
+}
+
+# Instantiate an attr of the test defined in global var `scenario`
+nixInstantiateTest() {
+    local attr=$1
+    shift
+    if [[ ${scenarioOverridesFile:-} ]]; then
+        local file="extraScenariosFile = \"$scenarioOverridesFile\";"
     else
-            hasFlakes=
+        local file=
     fi
+    nixInstantiate "(pkgs.getTest { name = \"$scenario\"; $file })$attr" "$@" >/dev/null
+}
+
+drv=
+# Sets global var `drv` to the gcroot link of the instantiated derivation
+nixInstantiate() {
+    local expr=$1
+    shift
+    makeTmpDir
+    drv="$tmpDir/drv"
+    nix-instantiate --add-root "$drv" -E "
+      let
+        pkgs = (builtins.getFlake \"git+file://$scriptDir/..\").legacyPackages.\${builtins.currentSystem};
+      in
+        $expr
+    " "$@"
+}
+
+nixBuild() {
+    local outLinkName=$1
+    shift
+    args=(--print-out-paths -L)
+    if [[ $outLinkPrefix ]]; then
+        args+=(--out-link "$outLinkPrefix-$outLinkName")
+    else
+        args+=(--no-link)
     fi
-    if [[ ! $hasFlakes ]]; then
+    nix build "${args[@]}" "$@"
-        echo "Skipping test '$testName'. Nix flake support is not enabled."
-        return 1
-    fi
 }
 
 flake() {
-    if ! checkFlakeSupport "flake"; then return; fi
-
     nix flake check "$scriptDir/.."
 }
 
 # Test generating module documentation for search.nixos.org
 nixosSearch() {
-    if ! checkFlakeSupport "nixosSearch"; then return; fi
-
     if [[ $outLinkPrefix ]]; then
         # Add gcroots for flake-info
         nix build "$scriptDir/nixos-search#flake-info" -o "$outLinkPrefix-flake-info"
@@ -285,26 +244,29 @@ nixosSearch() {
 }
 
 # A basic subset of tests to keep the total runtime within
-# manageable bounds (<4 min on desktop systems).
+# manageable bounds.
 # These are also run on the CI server.
-basic() {
+basic=(
-    scenario=default buildTest "$@"
+    default
-    scenario=netns buildTest "$@"
+    netns
-    scenario=netnsRegtest buildTest "$@"
+    netnsRegtest
-}
+)
+basic() { buildTests basic "$@"; }
 
 # All tests that only consist of building a nix derivation.
-# Their output is cached in /nix/store.
+# shellcheck disable=2034
-buildable() {
+buildable=(
-    basic "$@"
+    "${basic[@]}"
-    scenario=full buildTest "$@"
+    full
-    scenario=regtest buildTest "$@"
+    regtest
-    scenario=hardened buildTest "$@"
+    hardened
-    scenario=clightningReplication buildTest "$@"
+    clightningReplication
-    scenario=lndPruned buildTest "$@"
+    lndPruned
-}
+)
+buildable() { buildTests buildable "$@"; }
 
 examples() {
+    # shellcheck disable=SC2016
     script='
       set -e
       runExample() { echo; echo Running example $1; ./$1; }
@@ -317,7 +279,6 @@ examples() {
 }
 
 shellcheck() {
-    if ! checkFlakeSupport "shellcheck"; then return; fi
     "$scriptDir/shellcheck.sh"
 }
 

test/tests.nix

@@ -1,17 +1,9 @@
 # Integration tests, can be run without internet access.
 
+lib:
 let
-  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+  # Included in all scenarios
-in
+  baseConfig = { config, pkgs, ... }: with lib; let
-
-{ extraScenarios ? { ... }: {}
-, pkgs ? import nixpkgs { config = {}; overlays = []; }
-}:
-with pkgs.lib;
-let
-  globalPkgs = pkgs;
-
-  baseConfig = { pkgs, config, ... }: let
     cfg = config.services;
     inherit (config.nix-bitcoin.lib.test) mkIfTest;
   in {
@@ -32,9 +24,6 @@ let
     };
 
     config = mkMerge [{
-      # Share the same pkgs instance among tests
-      nixpkgs.pkgs = mkDefault globalPkgs;
-
       environment.systemPackages = mkMerge (with pkgs; [
         # Needed to test macaroon creation
         (mkIfTest "btcpayserver" [ openssl xxd ])
@@ -176,8 +165,9 @@ let
     ];
   };
 
-  scenarios = {
+  scenarios = with lib; {
-    base = baseConfig; # Included in all scenarios
+    # Included in all scenarios by ./lib/make-test.nix
+    base = baseConfig;
 
     default = scenarios.secureNode;
 
@@ -273,7 +263,7 @@ let
       environment.systemPackages = [ pkgs.fping ];
     };
 
-    regtestBase = { config, ... }: {
+    regtestBase = { config, pkgs, ... }: {
       tests.regtest = true;
       test.data.num_blocks = 100;
 
@@ -323,9 +313,10 @@ let
       services.lnd.enable = true;
       services.bitcoind.prune = 1000;
     };
+  };
 
-    ## Examples / debug helper
+  ## Example scenarios that showcase extra features
-
+  exampleScenarios = with lib; {
     # Run a selection of tests in scenario 'netns'
     selectedTests = {
       imports = [ scenarios.netns ];
@@ -342,40 +333,82 @@ let
       # See ./lib/test-lib.nix for a description
       test.container.exposeLocalhost = true;
     };
-
-    adhoc = {
-      # <Add your config here>
-      # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios.
-    };
   };
+in {
+  inherit scenarios;
 
-  overrides = builtins.getEnv "scenarioOverridesFile";
+  pkgs = flake: pkgs: rec {
-  extraScenarios' = (if (overrides != "") then import overrides else extraScenarios) {
+    # A basic test using the nix-bitcoin test framework
-    inherit scenarios pkgs;
+    makeTestBasic = import ./lib/make-test.nix flake pkgs makeTestVM;
-    inherit (pkgs) lib;
-  };
-  allScenarios = scenarios // extraScenarios';
 
-  makeTest = name: config:
+    # Wraps `makeTest` in NixOS' testing-python.nix so that the drv includes the
-    makeTest' name {
+    # log output and the test driver
+    makeTestVM = import ./lib/make-test-vm.nix pkgs;
+
+    # A test using the nix-bitcoin test framework, with some helpful defaults
+    makeTest = { name ? "nix-bitcoin-test", config }:
+      makeTestBasic {
+        inherit name;
+        config = {
           imports = [
-        allScenarios.base
+            scenarios.base
             config
           ];
-    };
+          # Share the same pkgs instance among tests
-  makeTest' = import ./lib/make-test.nix pkgs;
+          nixpkgs.pkgs = pkgs.lib.mkDefault pkgs;
-
-  tests = builtins.mapAttrs makeTest allScenarios // {
-    clightningReplication.vm = import ./clightning-replication.nix {
-      inherit pkgs;
-      inherit (pkgs.stdenv) system;
         };
       };
 
-  getTest = name: tests.${name} or (makeTest name {
+    # A test using the nix-bitcoin test framework, with defaults specific to nix-bitcoin
+    makeTestNixBitcoin = { name, config }:
+      makeTest {
+        name = "nix-bitcoin-${name}";
+        config = {
+          imports = [ config ];
+          test.shellcheckServices.sourcePrefix = toString ./..;
+        };
+      };
+
+    makeTests = scenarios: let
+      mainTests = builtins.mapAttrs (name: config:
+        makeTestNixBitcoin { inherit name config; }
+      ) scenarios;
+    in
+      {
+        clightningReplication = import ./clightning-replication.nix makeTestVM pkgs;
+      } // mainTests;
+
+    tests = makeTests scenarios;
+
+    ## Helper for ./run-tests.sh
+
+    getTest = { name, extraScenariosFile ? null }:
+      let
+        tests = makeTests (scenarios // (
+          lib.optionalAttrs (extraScenariosFile != null)
+            (import extraScenariosFile {
+              inherit scenarios lib pkgs;
+              nix-bitcoin = flake;
+            })
+        ));
+      in
+        tests.${name} or (makeTestNixBitcoin {
+          inherit name;
+          config = {
             services.${name}.enable = true;
+          };
         });
-in
+
-  tests // {
+    instantiateTests = testNames:
-    inherit getTest;
+      let
-  }
+        testNames' = lib.splitString " " testNames;
+      in
+        map (name:
+          let
+            test = tests.${name};
+          in
+            builtins.seq (builtins.trace "Evaluating test '${name}'" test.outPath)
+              test
+        ) testNames';
+  };
+}