nix-bitcoin/test/run-tests.sh
Erik Arvstedt edbaeb9813
tests: define tests via flake
Advantages:
- Pure test evaluations
- The test framework can now be used by flakes that extend nix-bitcoin
- Most features of `run-tests.sh` are now accessible via `nix build`/`nix run`.
  We keep `run-tests.sh` for advanced features like `scenarioOverridesFile` and adhoc scenarios.

Other changes:
- `run-tests.sh` now builds aggregate VM tests like `basic` or
  `buildable` by creating all VMs in a single evaluation.
  This speeds up the tests and eases debugging by separating the eval and build steps.
- Use the new `nix` CLI which has improved build output logging
  by prefixing output lines with the origin drv name.
2022-11-03 23:08:06 +01:00

312 lines
7.9 KiB
Bash
Executable File

#!/usr/bin/env bash
# Modules integration test runner.
# The tests (defined in ./tests.nix) use the NixOS testing framework and are executed in a VM.
#
# Usage:
# Run all tests
# ./run-tests.sh
#
# Test specific scenario
# ./run-tests.sh --scenario|-s <scenario>
#
# - When <scenario> contains a space, <scenario> is treated as nix code defining
# a scenario. It is evaluated in the same context as other scenarios in ./tests.nix
#
# Example:
# ./run-tests.sh -s "{ nix-bitcoin.nodeinfo.enable = true; }" container --run c nodeinfo
#
# - When <scenario> does not name a scenario, the test is run with an adhoc scenario
# where services.<scenario> is enabled.
#
# Example:
# ./run-tests.sh -s electrs
#
# Run test(s) and link results to avoid garbage collection
# ./run-tests.sh [--scenario <scenario>] --out-link-prefix /tmp/nix-bitcoin-test
#
# Pass extra args to nix-build
# ./run-tests.sh build --builders 'ssh://mybuildhost - - 15'
#
# Run interactive test debugging
# ./run-tests.sh [--scenario <scenario>] debug
#
# This starts the testing VM and drops you into a Python REPL where you can
# manually execute the tests from ./tests.py
#
# Run a test scenario in a container
# sudo ./run-tests.sh [--scenario <scenario>] container
#
# This is useful for quick experiments; containers start much faster than VMs.
# Running the Python test suite in containers is not yet supported.
# For now, creating NixOS containers requires root permissions.
# See ./lib/make-container.sh for a complete documentation.
#
# Run a test scenario in a regular NixOS VM.
# No tests are executed, the machine's serial console is attached to your terminal.
# ./run-tests.sh [--scenario <scenario>] vm
#
# This is useful for directly exploring a test configuration without the
# intermediate Python REPL layer.
# Run command 'q' inside the machine for instant poweroff.
#
# Run tests from a snapshot copy of the source files
# ./run-tests.sh --copy-src|-c ...
#
# This allows you to continue editing the nix-bitcoin sources while tests are running
# and reading source files.
# Files are copied to /tmp, a caching scheme helps minimizing copies.
#
# To add custom scenarios, set the environment variable `scenarioOverridesFile`.
set -eo pipefail
scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
args=("$@")
scenario=
outLinkPrefix=
while :; do
case $1 in
--scenario|-s)
if [[ $2 ]]; then
scenario=$2
shift
shift
else
>&2 echo "Error: $1 requires an argument."
exit 1
fi
;;
--out-link-prefix|-o)
if [[ $2 ]]; then
outLinkPrefix=$2
shift
shift
else
>&2 echo "Error: $1 requires an argument."
exit 1
fi
;;
--copy-src|-c)
shift
if [[ ! $_nixBitcoinInCopiedSrc ]]; then
. "$scriptDir/lib/copy-src.sh"
exit
fi
;;
*)
break
esac
done
tmpDir=
# Sets global var `tmpDir`
makeTmpDir() {
if [[ ! $tmpDir ]]; then
tmpDir=$(mktemp -d /tmp/nix-bitcoin-tests.XXX)
# shellcheck disable=SC2064
trap "rm -rf '$tmpDir'" EXIT
fi
}
# Support explicit scenario definitions
if [[ $scenario = *' '* ]]; then
makeTmpDir
export scenarioOverridesFile=$tmpDir/scenario-overrides.nix
echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
scenario=tmp
fi
# Run the test. No temporary files are left on the host system.
run() {
makeTmpDir
buildTestAttr .run --out-link "$tmpDir/run-vm"
NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/run-vm/bin/run-vm" "$@"
}
debug() {
run --debug
}
container() {
local nixosContainer
if ! nixosContainer=$(type -p nixos-container) \
|| grep -q '"/etc/nixos-containers"' "$nixosContainer"; then
local attr=container
else
# NixOS with `system.stateVersion` <22.05
local attr=containerLegacy
fi
echo "Building container"
makeTmpDir
export container=$tmpDir/container
buildTestAttr ".$attr" --out-link "$container"
export scriptDir
"$scriptDir/lib/make-container.sh" "$@"
}
# Run a regular NixOS VM
vm() {
makeTmpDir
buildTestAttr .vm --out-link "$tmpDir/vm"
NIX_BITCOIN_VM_DATADIR=$tmpDir "$tmpDir/vm/bin/run-vm-in-tmpdir"
}
# Run the test by building the test derivation
buildTest() {
buildTestAttr "" "$@"
}
evalTest() {
nixInstantiateTest "" "$@"
# Print out path
nix-store -q "$drv"
# Print drv path
realpath "$drv"
}
buildTestAttr() {
local attr=$1
shift
# TODO-EXTERNAL:
# Simplify and switch to pure build when `nix build` can build flake function outputs
nixInstantiateTest "$attr"
nixBuild "$scenario" "$drv" "$@"
}
buildTests() {
local -n tests=$1
shift
# TODO-EXTERNAL:
# Simplify and switch to pure build when `nix build` can instantiate flake function outputs
# shellcheck disable=SC2207
drvs=($(nixInstantiate "pkgs.instantiateTests \"${tests[*]}\""))
for i in "${!tests[@]}"; do
testName=${tests[$i]}
drv=${drvs[$i]}
echo
echo "Building test '$testName'"
nixBuild "$testName" "$drv" "$@"
done
}
# Instantiate an attr of the test defined in global var `scenario`
nixInstantiateTest() {
local attr=$1
shift
if [[ ${scenarioOverridesFile:-} ]]; then
local file="extraScenariosFile = \"$scenarioOverridesFile\";"
else
local file=
fi
nixInstantiate "(pkgs.getTest { name = \"$scenario\"; $file })$attr" "$@" >/dev/null
}
drv=
# Sets global var `drv` to the gcroot link of the instantiated derivation
nixInstantiate() {
local expr=$1
shift
makeTmpDir
drv="$tmpDir/drv"
nix-instantiate --add-root "$drv" -E "
let
pkgs = (builtins.getFlake \"git+file://$scriptDir/..\").legacyPackages.\${builtins.currentSystem};
in
$expr
" "$@"
}
nixBuild() {
local outLinkName=$1
shift
args=(--print-out-paths -L)
if [[ $outLinkPrefix ]]; then
args+=(--out-link "$outLinkPrefix-$outLinkName")
else
args+=(--no-link)
fi
nix build "${args[@]}" "$@"
}
flake() {
nix flake check "$scriptDir/.."
}
# Test generating module documentation for search.nixos.org
nixosSearch() {
if [[ $outLinkPrefix ]]; then
# Add gcroots for flake-info
nix build "$scriptDir/nixos-search#flake-info" -o "$outLinkPrefix-flake-info"
fi
"$scriptDir/nixos-search/flake-info-sandboxed.sh"
}
# A basic subset of tests to keep the total runtime within
# manageable bounds.
# These are also run on the CI server.
basic=(
default
netns
netnsRegtest
)
basic() { buildTests basic "$@"; }
# All tests that only consist of building a nix derivation.
# shellcheck disable=2034
buildable=(
"${basic[@]}"
full
regtest
hardened
clightningReplication
lndPruned
)
buildable() { buildTests buildable "$@"; }
examples() {
# shellcheck disable=SC2016
script='
set -e
runExample() { echo; echo Running example $1; ./$1; }
runExample deploy-container.sh
runExample deploy-container-minimal.sh
runExample deploy-qemu-vm.sh
runExample deploy-krops.sh
'
(cd "$scriptDir/../examples" && nix-shell --run "$script")
}
shellcheck() {
"$scriptDir/shellcheck.sh"
}
all() {
buildable "$@"
shellcheck
examples
flake
nixosSearch
}
# An alias for buildTest
build() {
buildTest "$@"
}
if [[ $# -gt 0 && $1 != -* ]]; then
# An explicit command was provided
command=$1
shift
if [[ $command == eval ]]; then
command=evalTest
fi
: ${scenario:=default}
elif [[ $scenario ]]; then
command=buildTest
else
command=basic
fi
$command "$@"