9ddf7864a4
When 'loop getparams' fails, jq gets no stdin and exits with code 0. Because -o pipefail is not enabled in the testing shell, the whole test command succeeds, although it should fail. Just test "loop getparams" instead and ignore its output.
353 lines
11 KiB
Python
353 lines
11 KiB
Python
from collections import OrderedDict
|
|
|
|
|
|
def succeed(*cmds):
|
|
"""Returns the concatenated output of all cmds"""
|
|
return machine.succeed(*cmds)
|
|
|
|
|
|
def assert_matches(cmd, regexp):
|
|
out = succeed(cmd)
|
|
if not re.search(regexp, out):
|
|
raise Exception(f"Pattern '{regexp}' not found in '{out}'")
|
|
|
|
|
|
def assert_full_match(cmd, regexp):
|
|
out = succeed(cmd)
|
|
if not re.fullmatch(regexp, out):
|
|
raise Exception(f"Pattern '{regexp}' doesn't match '{out}'")
|
|
|
|
|
|
def log_has_string(unit, str):
|
|
return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
|
|
|
|
|
|
def assert_no_failure(unit):
|
|
"""Unit should not have failed since the system is running"""
|
|
machine.fail(log_has_string(unit, "Failed with result"))
|
|
|
|
|
|
def assert_running(unit):
|
|
with machine.nested(f"waiting for unit: {unit}"):
|
|
machine.wait_for_unit(unit)
|
|
assert_no_failure(unit)
|
|
|
|
|
|
def wait_for_open_port(address, port):
|
|
def is_port_open(_):
|
|
status, _ = machine.execute(f"nc -z {address} {port}")
|
|
return status == 0
|
|
|
|
with log.nested(f"Waiting for TCP port {address}:{port}"):
|
|
retry(is_port_open)
|
|
|
|
|
|
### Test runner
|
|
|
|
tests = OrderedDict()
|
|
|
|
|
|
def test(name):
|
|
def x(fn):
|
|
tests[name] = fn
|
|
|
|
return x
|
|
|
|
|
|
def run_tests():
|
|
enabled = enabled_tests.copy()
|
|
to_run = []
|
|
for test in tests:
|
|
if test in enabled:
|
|
enabled.remove(test)
|
|
to_run.append(test)
|
|
if enabled:
|
|
raise RuntimeError(f"The following tests are enabled but not defined: {enabled}")
|
|
machine.connect() # Visually separate boot output from the test output
|
|
for test in to_run:
|
|
with log.nested(f"test: {test}"):
|
|
tests[test]()
|
|
|
|
|
|
def run_test(test):
|
|
tests[test]()
|
|
|
|
|
|
### Tests
|
|
# All tests are executed in the order they are defined here
|
|
|
|
|
|
@test("security")
|
|
def _():
|
|
assert_running("setup-secrets")
|
|
# Unused secrets should be inaccessible
|
|
succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
|
|
|
|
if "secure-node" in enabled_tests:
|
|
# Access to '/proc' should be restricted
|
|
machine.succeed("grep -Fq hidepid=2 /proc/mounts")
|
|
|
|
machine.wait_for_unit("bitcoind")
|
|
# `systemctl status` run by unprivileged users shouldn't leak cgroup info
|
|
assert_matches(
|
|
"sudo -u electrs systemctl status bitcoind 2>&1 >/dev/null",
|
|
"Failed to dump process list for 'bitcoind.service', ignoring: Access denied",
|
|
)
|
|
# The 'operator' with group 'proc' has full access
|
|
assert_full_match("sudo -u operator systemctl status bitcoind 2>&1 >/dev/null", "")
|
|
|
|
|
|
@test("bitcoind")
|
|
def _():
|
|
assert_running("bitcoind")
|
|
machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
|
|
assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
|
|
# RPC access for user 'public' should be restricted
|
|
machine.fail(
|
|
"bitcoin-cli -rpcuser=public -rpcpassword=$(cat /secrets/bitcoin-rpcpassword-public) stop"
|
|
)
|
|
machine.wait_until_succeeds(
|
|
log_has_string("bitcoind", "RPC User public not allowed to call method stop")
|
|
)
|
|
|
|
|
|
@test("electrs")
|
|
def _():
|
|
assert_running("electrs")
|
|
wait_for_open_port(ip("electrs"), 4224) # prometeus metrics provider
|
|
# Check RPC connection to bitcoind
|
|
machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
|
|
|
|
|
|
# Impure: Stops electrs
|
|
# Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages
|
|
@test("stop-electrs")
|
|
def _():
|
|
succeed("systemctl stop electrs")
|
|
|
|
|
|
@test("liquidd")
|
|
def _():
|
|
assert_running("liquidd")
|
|
machine.wait_until_succeeds("elements-cli getnetworkinfo")
|
|
assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
|
|
succeed("su operator -c 'liquidswap-cli --help'")
|
|
|
|
|
|
@test("clightning")
|
|
def _():
|
|
assert_running("clightning")
|
|
assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
|
|
|
|
|
|
@test("lnd")
|
|
def _():
|
|
assert_running("lnd")
|
|
assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
|
|
assert_no_failure("lnd")
|
|
|
|
|
|
@test("lightning-loop")
|
|
def _():
|
|
assert_running("lightning-loop")
|
|
assert_matches("su operator -c 'loop --version'", "version")
|
|
# Check that lightning-loop fails with the right error, making sure
|
|
# lightning-loop can connect to lnd
|
|
machine.wait_until_succeeds(
|
|
log_has_string(
|
|
"lightning-loop",
|
|
"Waiting for lnd to be fully synced to its chain backend, this might take a while",
|
|
)
|
|
)
|
|
|
|
|
|
@test("btcpayserver")
|
|
def _():
|
|
assert_running("nbxplorer")
|
|
machine.wait_until_succeeds(log_has_string("nbxplorer", "BTC: RPC connection successful"))
|
|
wait_for_open_port(ip("nbxplorer"), 24444)
|
|
assert_running("btcpayserver")
|
|
machine.wait_until_succeeds(log_has_string("btcpayserver", "Listening on"))
|
|
wait_for_open_port(ip("btcpayserver"), 23000)
|
|
# test lnd custom macaroon
|
|
assert_matches(
|
|
"sudo -u btcpayserver curl -s --cacert /secrets/lnd-cert "
|
|
'--header "Grpc-Metadata-macaroon: $(xxd -ps -u -c 1000 /run/lnd/btcpayserver.macaroon)" '
|
|
f"-X GET https://{ip('lnd')}:8080/v1/getinfo | jq",
|
|
'"version"',
|
|
)
|
|
|
|
|
|
@test("spark-wallet")
|
|
def _():
|
|
assert_running("spark-wallet")
|
|
wait_for_open_port(ip("spark-wallet"), 9737)
|
|
spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
|
|
assert_matches(f"curl -s {spark_auth}@{ip('spark-wallet')}:9737", "Spark")
|
|
|
|
|
|
@test("lightning-charge")
|
|
def _():
|
|
assert_running("lightning-charge")
|
|
wait_for_open_port(ip("lightning-charge"), 9112)
|
|
machine.wait_until_succeeds(f"nc -z {ip('lightning-charge')} 9112")
|
|
charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
|
|
assert_matches(
|
|
f"curl -s api-token:{charge_auth}@{ip('lightning-charge')}:9112/info | jq", '"id"'
|
|
)
|
|
|
|
|
|
@test("nanopos")
|
|
def _():
|
|
assert_running("nanopos")
|
|
wait_for_open_port(ip("nanopos"), 9116)
|
|
assert_matches(f"curl {ip('nanopos')}:9116", "tshirt")
|
|
|
|
|
|
@test("joinmarket")
|
|
def _():
|
|
assert_running("joinmarket")
|
|
machine.wait_until_succeeds(
|
|
log_has_string("joinmarket", "P2EPDaemonServerProtocolFactory starting on 27184")
|
|
)
|
|
|
|
|
|
@test("joinmarket-yieldgenerator")
|
|
def _():
|
|
machine.wait_until_succeeds(
|
|
log_has_string("joinmarket-yieldgenerator", "Failure to get blockheight",)
|
|
)
|
|
|
|
|
|
@test("secure-node")
|
|
def _():
|
|
assert_running("onion-chef")
|
|
|
|
# FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
|
|
# to incomplete unit dependencies.
|
|
# 'create-web-index' implicitly tests 'nodeinfo'.
|
|
machine.wait_for_unit("create-web-index")
|
|
assert_running("nginx")
|
|
wait_for_open_port(ip("nginx"), 80)
|
|
assert_matches(f"curl {ip('nginx')}", "nix-bitcoin")
|
|
assert_matches(f"curl -L {ip('nginx')}/store", "tshirt")
|
|
|
|
|
|
# Run this test before the following tests that shut down services
|
|
# (and their corresponding network namespaces).
|
|
@test("netns-isolation")
|
|
def _():
|
|
def get_ips(services):
|
|
enabled = enabled_tests.intersection(services)
|
|
return " ".join(ip(service) for service in enabled)
|
|
|
|
def assert_reachable(src, dests):
|
|
dest_ips = get_ips(dests)
|
|
if src in enabled_tests and dest_ips:
|
|
machine.succeed(f"ip netns exec nb-{src} fping -c1 -t100 {dest_ips}")
|
|
|
|
def assert_unreachable(src, dests):
|
|
dest_ips = get_ips(dests)
|
|
if src in enabled_tests and dest_ips:
|
|
machine.fail(
|
|
# This fails when no host is reachable within 100 ms
|
|
f"ip netns exec nb-{src} fping -c1 -t100 --reachable=1 {dest_ips}"
|
|
)
|
|
|
|
# These reachability tests are non-exhaustive
|
|
assert_reachable("bitcoind", ["clightning", "lnd", "liquidd"])
|
|
assert_unreachable("bitcoind", ["btcpayserver", "spark-wallet", "lightning-loop"])
|
|
assert_unreachable("btcpayserver", ["bitcoind", "lightning-loop", "liquidd"])
|
|
|
|
if "joinmarket" in enabled_tests:
|
|
# netns-exec should drop capabilities
|
|
assert_full_match(
|
|
"su operator -c 'netns-exec nb-joinmarket capsh --print | grep Current'", "Current: =\n"
|
|
)
|
|
|
|
if "clightning" in enabled_tests:
|
|
# netns-exec should fail for unauthorized namespaces
|
|
machine.fail("netns-exec nb-clightning ip a")
|
|
|
|
# netns-exec should only be executable by the operator user
|
|
machine.fail("sudo -u clightning netns-exec nb-bitcoind ip a")
|
|
|
|
|
|
# Impure: stops bitcoind (and dependent services)
|
|
@test("backups")
|
|
def _():
|
|
succeed("systemctl stop bitcoind")
|
|
succeed("systemctl start duplicity")
|
|
machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded."))
|
|
run_duplicity = "export $(cat /secrets/backup-encryption-env); duplicity"
|
|
# Files in backup and /var/lib should be identical
|
|
assert_matches(
|
|
f"{run_duplicity} verify --archive-dir /var/lib/duplicity file:///var/lib/localBackups /var/lib",
|
|
"0 differences found",
|
|
)
|
|
# Backup should include important files
|
|
files = succeed(f"{run_duplicity} list-current-files file:///var/lib/localBackups")
|
|
assert "var/lib/clightning/bitcoin/hsm_secret" in files
|
|
assert "secrets/lnd-seed-mnemonic" in files
|
|
assert "secrets/jm-wallet-seed" in files
|
|
assert "var/lib/bitcoind/wallet.dat" in files
|
|
assert "var/backup/postgresql/btcpaydb.sql.gz" in files
|
|
|
|
|
|
# Impure: restarts services
|
|
@test("banlist-and-restart")
|
|
def _():
|
|
machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
|
|
assert_no_failure("bitcoind-import-banlist")
|
|
|
|
# Current time in µs
|
|
pre_restart = succeed("date +%s.%6N").rstrip()
|
|
|
|
# Sanity-check system by restarting all services
|
|
succeed(
|
|
"systemctl restart bitcoind clightning lnd lightning-loop spark-wallet lightning-charge nanopos liquidd"
|
|
)
|
|
|
|
# Now that the bitcoind restart triggered a banlist import restart, check that
|
|
# re-importing already banned addresses works
|
|
machine.wait_until_succeeds(
|
|
log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
|
|
)
|
|
assert_no_failure("bitcoind-import-banlist")
|
|
|
|
|
|
@test("regtest")
|
|
def _():
|
|
if "electrs" in enabled_tests:
|
|
machine.wait_until_succeeds(log_has_string("electrs", "BlockchainInfo"))
|
|
get_block_height_cmd = (
|
|
"""echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'"""
|
|
f" | nc -N {ip('electrs')} 50001 | jq -M .result.height"
|
|
)
|
|
assert_full_match(get_block_height_cmd, "10\n")
|
|
if "clightning" in enabled_tests:
|
|
machine.wait_until_succeeds(
|
|
"[[ $(sudo -u operator lightning-cli getinfo | jq -M .blockheight) == 10 ]]"
|
|
)
|
|
if "lnd" in enabled_tests:
|
|
machine.wait_until_succeeds(
|
|
"[[ $(sudo -u operator lncli getinfo | jq -M .block_height) == 10 ]]"
|
|
)
|
|
if "lightning-loop" in enabled_tests:
|
|
machine.wait_until_succeeds(
|
|
log_has_string("lightning-loop", "Starting event loop at height 10")
|
|
)
|
|
succeed("sudo -u operator loop getparams")
|
|
|
|
|
|
if "netns-isolation" in enabled_tests:
|
|
|
|
def ip(name):
|
|
return test_data["netns"][name]["address"]
|
|
|
|
|
|
else:
|
|
|
|
def ip(_):
|
|
return "127.0.0.1"
|