tests: allow running integration tests with different configurations

test/run-tests.sh

@@ -5,19 +5,52 @@
 #
 # Usage:
 #   Run test
-#   ./run-tests.sh
+#   ./run-tests.sh --scenario <scenario>
 #
 #   Run test and save result to avoid garbage collection
-#   ./run-tests.sh build --out-link /tmp/nix-bitcoin-test
+#   ./run-tests.sh --scenario <scenario> build --out-link /tmp/nix-bitcoin-test
 #
 #   Run interactive test debugging
-#   ./run-tests.sh debug
+#   ./run-tests.sh --scenario <scenario> debug
 #
 #   This starts the testing VM and drops you into a Python REPL where you can
 #   manually execute the tests from ./test-script.py
 
 set -eo pipefail
 
+die() {
+    printf '%s\n' "$1" >&2
+    exit 1
+}
+
+# Initialize all the option variables.
+# This ensures we are not contaminated by variables from the environment.
+scenario=
+
+while :; do
+    case $1 in
+	--scenario)
+	    if [ "$2" ]; then
+		scenario=$2
+		shift
+	    else
+		die 'ERROR: "--scenario" requires a non-empty option argument.'
+	    fi
+	    ;;
+	-?*)
+	    printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2
+	    ;;
+	*)
+	    break
+    esac
+
+    shift
+done
+
+if [[ -z $scenario ]]; then
+    die 'ERROR: "--scenario" is required'
+fi
+
 numCPUs=${numCPUs:-$(nproc)}
 # Min. 800 MiB needed to avoid 'out of memory' errors
 memoryMiB=${memoryMiB:-2048}
@@ -32,7 +65,7 @@ run() {
     export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
     trap "rm -rf $TMPDIR" EXIT
 
-    nix-build --out-link $TMPDIR/driver "$scriptDir/test.nix" -A driver
+    nix-build --out-link $TMPDIR/driver -E "import \"$scriptDir/test.nix\" { scenario = \"$scenario\"; }" -A driver
 
     # Variable 'tests' contains the Python code that is executed by the driver on startup
     if [[ $1 == --interactive ]]; then
@@ -95,7 +128,7 @@ exprForCI() {
 
 vmTestNixExpr() {
   cat <<EOF
-    (import "$scriptDir/test.nix" {}).overrideAttrs (old: rec {
+    (import "$scriptDir/test.nix" { scenario = "$scenario"; } {}).overrideAttrs (old: rec {
       buildCommand = ''
         export QEMU_OPTS="-smp $numCPUs -m $memoryMiB"
         echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"

test/scenarios/default.py

@@ -0,0 +1,77 @@
+### Tests
+
+assert_running("setup-secrets")
+# Unused secrets should be inaccessible
+succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
+
+assert_running("bitcoind")
+machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
+assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
+
+assert_running("electrs")
+machine.wait_for_open_port(4224)  # prometeus metrics provider
+# Check RPC connection to bitcoind
+machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
+assert_running("nginx")
+# SSL stratum server via nginx. Only check for open port, no content is served here
+# as electrs isn't ready.
+machine.wait_for_open_port(50003)
+# Stop electrs from spamming the test log with 'wait for bitcoind sync' messages
+succeed("systemctl stop electrs")
+
+assert_running("liquidd")
+machine.wait_until_succeeds("elements-cli getnetworkinfo")
+assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
+succeed("su operator -c 'liquidswap-cli --help'")
+
+assert_running("clightning")
+assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+
+assert_running("spark-wallet")
+spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
+machine.wait_for_open_port(9737)
+assert_matches(f"curl -s {spark_auth}@localhost:9737", "Spark")
+
+assert_running("lightning-charge")
+charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
+machine.wait_for_open_port(9112)
+assert_matches(f"curl -s api-token:{charge_auth}@localhost:9112/info | jq", '"id"')
+
+assert_running("nanopos")
+machine.wait_for_open_port(9116)
+assert_matches("curl localhost:9116", "tshirt")
+
+assert_running("onion-chef")
+
+# FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
+# to incomplete unit dependencies.
+# 'create-web-index' implicitly tests 'nodeinfo'.
+machine.wait_for_unit("create-web-index")
+machine.wait_for_open_port(80)
+assert_matches("curl localhost", "nix-bitcoin")
+assert_matches("curl -L localhost/store", "tshirt")
+
+machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
+assert_no_failure("bitcoind-import-banlist")
+
+### Additional tests
+
+# Current time in µs
+pre_restart = succeed("date +%s.%6N").rstrip()
+
+# Sanity-check system by restarting all services
+succeed("systemctl restart bitcoind clightning spark-wallet lightning-charge nanopos liquidd")
+
+# Now that the bitcoind restart triggered a banlist import restart, check that
+# re-importing already banned addresses works
+machine.wait_until_succeeds(
+    log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
+)
+assert_no_failure("bitcoind-import-banlist")
+
+### Test lnd
+
+succeed("systemctl stop nanopos lightning-charge spark-wallet clightning")
+succeed("systemctl start lnd")
+assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
+assert_no_failure("lnd")

test/scenarios/lib.py

@@ -0,0 +1,34 @@
+def succeed(*cmds):
+    """Returns the concatenated output of all cmds"""
+    return machine.succeed(*cmds)
+
+
+def assert_matches(cmd, regexp):
+    out = succeed(cmd)
+    if not re.search(regexp, out):
+        raise Exception(f"Pattern '{regexp}' not found in '{out}'")
+
+
+def assert_matches_exactly(cmd, regexp):
+    out = succeed(cmd)
+    if not re.fullmatch(regexp, out):
+        raise Exception(f"Pattern '{regexp}' doesn't match '{out}'")
+
+
+def log_has_string(unit, str):
+    return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
+
+
+def assert_no_failure(unit):
+    """Unit should not have failed since the system is running"""
+    machine.fail(log_has_string(unit, "Failed with result"))
+
+
+def assert_running(unit):
+    machine.wait_for_unit(unit)
+    assert_no_failure(unit)
+
+
+# Don't execute the following test suite when this script is running in interactive mode
+if "is_interactive" in vars():
+    raise Exception()

test/scenarios/withnetns.py

@@ -1,38 +1,3 @@
-def succeed(*cmds):
-    """Returns the concatenated output of all cmds"""
-    return machine.succeed(*cmds)
-
-
-def assert_matches(cmd, regexp):
-    out = succeed(cmd)
-    if not re.search(regexp, out):
-        raise Exception(f"Pattern '{regexp}' not found in '{out}'")
-
-
-def assert_matches_exactly(cmd, regexp):
-    out = succeed(cmd)
-    if not re.fullmatch(regexp, out):
-        raise Exception(f"Pattern '{regexp}' doesn't match '{out}'")
-
-
-def log_has_string(unit, str):
-    return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
-
-
-def assert_no_failure(unit):
-    """Unit should not have failed since the system is running"""
-    machine.fail(log_has_string(unit, "Failed with result"))
-
-
-def assert_running(unit):
-    machine.wait_for_unit(unit)
-    assert_no_failure(unit)
-
-
-# Don't execute the following test suite when this script is running in interactive mode
-if "is_interactive" in vars():
-    raise Exception()
-
 # netns IP addresses
 bitcoind_ip = "169.254.1.12"
 clightning_ip = "169.254.1.13"

test/test.nix

@@ -1,5 +1,12 @@
 # Integration test, can be run without internet access.
 
+{ scenario ? "default" }:
+
+let
+  netns-isolation = builtins.getAttr scenario { default = false; withnetns = true; };
+  testScriptFilename = builtins.getAttr scenario { default = ./scenarios/default.py; withnetns = ./scenarios/withnetns.py; };
+in
+
 import ./make-test.nix rec {
   name = "nix-bitcoin";
 
@@ -16,7 +23,7 @@ import ./make-test.nix rec {
       # hardened
     ];
 
-    nix-bitcoin.netns-isolation.enable = mkForce true;
+    nix-bitcoin.netns-isolation.enable = mkForce netns-isolation;
 
     services.bitcoind.extraConfig = mkForce "connect=0";
 
@@ -48,6 +55,5 @@ import ./make-test.nix rec {
       install -o nobody -g nogroup -m777 <(:) /secrets/dummy
     '';
   };
-
-  testScript = builtins.readFile ./test-script.py;
+  testScript = builtins.readFile ./scenarios/lib.py + "\n\n" + builtins.readFile testScriptFilename;
 }