diff --git a/README.md b/README.md index 75147e3..1058589 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,10 @@ The nix-bitcoin security fund is a 2 of 3 bitcoin multisig address open for dona security researchers who discover vulnerabilities in nix-bitcoin or its upstream dependencies.\ See [Security Fund](./SECURITY.md#nix-bitcoin-security-fund) for details. +Developing +--- +See [dev/README](./dev/README.md). + Troubleshooting --- If you are having problems with nix-bitcoin check the [FAQ](docs/faq.md) or submit an issue.\ diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..9b93a5c --- /dev/null +++ b/dev/README.md @@ -0,0 +1,104 @@ +This directory contains docs and helper scripts for developing and debugging: + +- [`dev.sh`](./dev.sh): misc dev helpers +- [`dev-features.sh`](./dev-features.sh): helpers for developing specific + nix-bitcoin features, like services +- [`topics`](./topics) features specific topics +- [`dev-scenarios.nix`](./dev-scenarios.nix): extra test scenarios used in the above scripts + +See also: [test/README.md](../test/README.md) + +## Run a dev shell + +There are two ways to run a dev shell: + +### 1. Run command `nix develop` + +This starts a shell with [`test/run-tests.sh`](../test/run-tests.sh) and +the scripts in dir [`helper`](../helper) added to `PATH`. + +### 2. Setup and start the `direnv` dev env + +This is an opinionated, [direnv](https://direnv.net/)-based dev env, optimized for developer experience. + +[`dev-env/create.sh`](./dev-env/create.sh) creates a git repo with the following contents: +- Dir `src` which contains the nix-bitcoin repo +- Dir `bin` for helper scripts +- File `scenarios.nix` for custom test scenarios +- File `.envrc` that defines a [direnv](https://direnv.net/) environment, + mainly for adding nix-bitcoin and helper scripts to `PATH` + +#### Installation + +1. [Install direnv](https://direnv.net/docs/installation.html).\ + If you use NixOS (and Bash as the default shell), just add the following to your system config: + ```nix + environment.systemPackages = [ pkgs.direnv ]; + programs.bash.interactiveShellInit = '' + eval "$(direnv hook bash)" + ''; + ``` + +2. Create the dev env: + ```bash + # Set up a dev environment in dir ~/dev/nix-bitcoin. + # The dir is created automatically. + ./dev-env/create.sh ~/dev/nix-bitcoin + + cd ~/dev/nix-bitcoin + + # Enable direnv + direnv allow + ``` + +3. Optional: Editor integration + - Add envrc support to your editor + - Setup your editor so you can easily execute lines or paragraphs from a shell script + file in a shell.\ + This simplifies using dev helper scripts like [`./dev.sh`](./dev.sh). + +#### Explore the dev env +```bash +# The direnv is automatically activated when visiting any subdir of ~/dev/nix-bitcoin +cd ~/dev/nix-bitcoin + +ls -al . bin lib + +# The direnv config file +cat .envrc + +# You can use this file to define extra scenarios +cat scenarios.nix + +# Binary `dev-run-tests` runs nix-bitcoin's `run-tests.sh` with extra scenarios from ./scenarios.nix +# Example: +# Run command `nodeinfo` in `myscenario` (defined in ./scenarios.nix) via a container +dev-run-tests -s myscenario container --run c nodeinfo + +# Equivalent (shorthand) +te -s myscenario container --run c nodeinfo + +# Run the tests for `myscenario` in a VM +te -s myscenario + +# Start an interactive shell inside a VM +te -s myscenario vm +``` + +See also: [test/README.md](../test/README.md) + +## Adding a new service + +It's easiest to use an existing service as a template: +- [electrs.nix](../modules/electrs.nix): a basic service +- [clightning.nix](../modules/clightning.nix): simple, but covers a few more features.\ + (A `cli` binary and a runtime-composed config to include secrets.) +- [rtl.nix](../modules/rtl.nix): includes a custom package, defined in [pkgs/rtl](../pkgs/rtl).\ + Most other services use packages that are already included in nixpkgs. + +## Switching to a new NixOS release + +- [flake.nix](../flake.nix): update `nixpkgs.url` +- [cirrus.yml](../.cirrus.yml): update toplevel container -> image attribute +- [examples/configuration.nix](../examples/configuration.nix): update `system.stateVersion` +- Treewide: check if any `TODO-EXTERNAL` comments can be resolved diff --git a/dev/dev-env/create.sh b/dev/dev-env/create.sh new file mode 100755 index 0000000..119ab6d --- /dev/null +++ b/dev/dev-env/create.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 +set -euo pipefail + +destDir=${1:-nix-bitcoin} + +scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) + +mkdir -p "$destDir/"{bin,lib} +cd "$destDir" + +if [[ ! -e src ]]; then + echo "Cloning fort-nix/nix-bitcoin" + git clone https://github.com/fort-nix/nix-bitcoin src +fi + +echo 'export root=$PWD +export src=$root/src +PATH_add bin +PATH_add src/helper' > .envrc + +if [[ ! -e scenarios.nix ]]; then + cp "$scriptDir/template-scenarios.nix" scenarios.nix +fi + +install -m 755 <( + echo '#!/usr/bin/env bash' + echo 'exec run-tests.sh --extra-scenarios "$root/scenarios.nix" "$@"' +) bin/dev-run-tests + +install -m 755 <( + echo '#!/usr/bin/env bash' + echo 'exec $root/src/test/run-tests.sh --out-link-prefix /tmp/nix-bitcoin/test "$@"' +) bin/run-tests.sh + +ln -sfn dev-run-tests bin/te + +## nix-bitcoin-firejail + +echo '# Add your shell config files here that should be accessible in the sandbox +whitelist ${HOME}/.bashrc +read-only ${HOME}/.bashrc' > lib/nix-bitcoin-firejail.conf + +install -m 755 <( + echo '#!/usr/bin/env bash' + echo '# A sandbox for running shells/binaries in an isolated environment:' + echo '# - The sandbox user is the calling user, with all capabilities dropped' + echo '# and with no way to gain new privileges (e.g. via `sudo`).' + echo '# - $HOME is bind-mounted to a dir that only contains shell config files and files required by direnv.' + echo '#' + echo '# You can modify the firejail env by editing `lib/nix-bitcoin-firejail.conf` in your dev env dir.' + echo 'exec firejail --profile="$root/lib/nix-bitcoin-firejail.conf" --profile="$root/src/dev/dev-env/nix-bitcoin-firejail.conf" "$@"' +) bin/nix-bitcoin-firejail + +echo "1" > lib/dev-env-version + +## git + +echo '/src' > .gitignore + +if [[ ! -e .git ]]; then + git init + git add . + git commit -a -m init +fi diff --git a/dev/dev-env/dev-shell.nix b/dev/dev-env/dev-shell.nix new file mode 100644 index 0000000..954284f --- /dev/null +++ b/dev/dev-env/dev-shell.nix @@ -0,0 +1,16 @@ +pkgs: + +pkgs.mkShell { + shellHook = '' + # A known rev from the master branch to test whether `nix develop` + # is called inside the nix-bitcoin repo + rev=5cafafd02777919c10e559b5686237fdefe920c2 + if git cat-file -e $rev &>/dev/null; then + root=$(git rev-parse --show-toplevel) + export PATH=$root/test:$root/helper:$PATH + else + echo 'Error: `nix develop` must be called inside the nix-bitcoin repo.' + exit 1 + fi + ''; +} diff --git a/dev/dev-env/nix-bitcoin-firejail.conf b/dev/dev-env/nix-bitcoin-firejail.conf new file mode 100644 index 0000000..477f48a --- /dev/null +++ b/dev/dev-env/nix-bitcoin-firejail.conf @@ -0,0 +1,25 @@ +include default.local +include globals.local + +include disable-common.inc +include disable-programs.inc + +caps.drop all +netfilter +noinput +nonewprivs +noroot +notv +novideo +protocol unix,inet,inet6 +seccomp + +## Enable features + +allow-debuggers + +# Enable direnv configs +whitelist ${HOME}/.config/direnv +read-only ${HOME}/.config/direnv +whitelist ${HOME}/.local/share/direnv +read-only ${HOME}/.local/share/direnv diff --git a/dev/dev-env/template-scenarios.nix b/dev/dev-env/template-scenarios.nix new file mode 100644 index 0000000..156ed40 --- /dev/null +++ b/dev/dev-env/template-scenarios.nix @@ -0,0 +1,20 @@ +{ pkgs, lib, scenarios, nix-bitcoin }: +with lib; +rec { + # For more examples, see `scenarios` and `exampleScenarios` in ./src/test/tests.nix + + template = { config, pkgs, lib, ... }: { + imports = [ + (nix-bitcoin + "/modules/presets/secure-node.nix") + scenarios.netnsBase + scenarios.regtestBase + ]; + test.container.enableWAN = true; + test.container.exposeLocalhost = true; + }; + + myscenario = { config, pkgs, lib, ... }: { + services.clightning.enable = true; + nix-bitcoin.nodeinfo.enable = true; + }; +} diff --git a/dev/dev-features.sh b/dev/dev-features.sh new file mode 100644 index 0000000..8eba0e8 --- /dev/null +++ b/dev/dev-features.sh @@ -0,0 +1,290 @@ +# shellcheck disable=SC2086,SC2154 + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Run tests +# See ../test/README.md for a tutorial +# and ../test/run-tests.sh for a complete documentation + +# Start a shell in a container +run-tests.sh -s electrs container + +# Run a command in a container. +# The container is deleted afterwards. +run-tests.sh -s electrs container --run c journalctl -u electrs + +# Run a bash command +run-tests.sh -s bitcoind container --run c bash -c "sleep 1; journalctl -u bitcoind" + +run-tests.sh -s '{ + imports = [ scenarios.regtestBase ]; + services.electrs.enable = true; +}' container --run c journalctl -u electrs + +run-tests.sh -s "{ + services.electrs.enable = true; + nix-bitcoin.nodeinfo.enable = true; +}" container --run c nodeinfo + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Get generic node infos + +# Start container shell +run-tests.sh -s bitcoind container + +# Run commands inside the shell: + +# The node's services +c systemctl status + +# Failed units +c systemctl list-units --failed + +# Analyze container boot performance +c systemd-analyze critical-chain + +# Listening TCP sockets +c netstat -nltp +# Listening sockets +c netstat -nlp + +# The container root filesystem +ls -al /var/lib/nixos-containers/nb-test + +# The container root filesystem on NixOS systems with stateVersion < 22.05 +ls -al /var/lib/containers/nb-test + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# bitcoind +run-tests.sh -s bitcoind container + +c systemctl status bitcoind +c systemctl cat bitcoind +c journalctl --output=short-precise -u bitcoind +ls -al /var/lib/nixos-containers/nb-test/var/lib/bitcoind +c bitcoin-cli getpeerinfo +c bitcoin-cli getnetworkinfo +c bitcoin-cli getblockchaininfo + +run-tests.sh -s '{ + imports = [ scenarios.regtestBase ]; + services.bitcoind.enable = true; +}' container + +address=$(c bitcoin-cli getnewaddress) +echo $address +c bitcoin-cli generatetoaddress 10 $address + +# Run bitcoind with network access +run-tests.sh -s "{ + test.container.enableWAN = true; + services.bitcoind.enable = true; +}" container --run c journalctl -u bitcoind -f + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# clightning +run-tests.sh -s clightning container + +c systemctl status clightning +c journalctl --output=short-precise -u clightning +c lightning-cli getinfo + +# Plugins +run-tests.sh -s "{ + services.clightning.enable = true; + test.features.clightningPlugins = true; +}" container + +c lightning-cli plugin list + +# Show plugin config +nix eval --raw .#makeTest --apply ' + makeTest: let + config = (makeTest { + config = { + services.clightning.enable = true; + test.features.clightningPlugins = true; + }; + }).nodes.machine; + in + config.services.clightning.extraConfig +' + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# clightning-rest +run-tests.sh -s clightning-rest container + +c systemctl status clightning-rest +c journalctl -u clightning-rest +c systemctl status clightning-rest-migrate-datadir + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# spark-wallet + +run-tests.sh -s "{ + services.spark-wallet.enable = true; + test.container.exposeLocalhost = true; +}" container + +c systemctl status spark-wallet +c journalctl -u spark-wallet + +sparkAuth=$(c cat /secrets/spark-wallet-login | grep -ohP '(?<=login=).*') +curl -v http://$sparkAuth@$ip:9737 +# Open in browser +runuser -u "$(logname)" -- xdg-open http://$sparkAuth@$ip:9737 + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# electrs + +run-tests.sh -s "{ + imports = [ scenarios.regtestBase ]; + services.electrs.enable = true; +}" container + +c systemctl status electrs +c systemctl cat electrs +c journalctl --output=short-precise -u electrs + +electrs_rpc() { + echo "$1" | c nc 127.0.0.1 50001 | head -1 | jq +} +electrs_rpc '{"method": "server.version", "id": 0, "params": ["electrum/3.3.8", "1.4"]}' +electrs_rpc '{"method": "blockchain.headers.subscribe", "id": 0}' + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# fulcrum + +run-tests.sh -s "{ + imports = [ scenarios.regtestBase ]; + services.fulcrum.enable = true; +}" container + +c systemctl status fulcrum +c systemctl cat fulcrum +c journalctl --output=short-precise -u fulcrum + +fulcrum_rpc() { + echo "$1" | c nc 127.0.0.1 50002 | head -1 | jq +} +fulcrum_rpc '{"method": "server.version", "id": 0, "params": ["electrum/3.3.8", "1.4"]}' +fulcrum_rpc '{"method": "blockchain.headers.subscribe", "id": 0}' + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# lnd +run-tests.sh -s lnd container +c systemctl status lnd +c journalctl -u lnd +c lncli getinfo + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# lightning-loop +run-tests.sh -s lightning-loop container +c systemctl status lightning-loop +c journalctl -u lightning-loop + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# btcpayserver +# https://docs.btcpayserver.org/Development/GreenFieldExample/ + +run-tests.sh -s btcpayserver-regtest container + +c systemctl status btcpayserver +c journalctl -u btcpayserver +c systemctl cat btcpayserver + +c systemctl status nbxplorer +c journalctl -u nbxplorer + +## Access the API +request() { + local type=$1 + local method=$2 + local body=$3 + shift; shift; shift + curl -sSL -H "Content-Type: application/json" -X $type --user "a@a.a:aaaaaa" \ + -d "$body" "$@" "$ip:23000/api/v1/$method" | jq +} +post() { + local method=$1 + local body=$2 + shift; shift + request post "$method" "$body" "$@" +} +get() { + local method=$1 + request get "$method" +} + +# Create new user +post users '{"email": "a@a.a", "password": "aaaaaa", "isAdministrator": true}' + +# Login with: +# user: a@a.a +# password: aaaaaa +runuser -u "$(logname)" -- xdg-open http://$ip:23000 + +# create store +post stores '{"name": "a", "defaultPaymentMethod": "BTC_LightningNetwork"}' +post stores '{"name": "a", "defaultPaymentMethod": "BTC"}' + +store=$(get stores | jq -r .[].id) +echo $store +get stores/$store +get stores/$store/payment-methods +get stores/$store/payment-methods/LightningNetwork + +# Connect to internal lightning node (internal API, doesn't work) +# Lightning must be manually setup via the webinterface. +post stores/$store/lightning/BTC/setup "" --data-raw 'LightningNodeType=Internal&ConnectionString=&command=save' + +nix run --inputs-from . nixpkgs#lynx -- --dump http://$ip:23000/embed/$store/BTC/ln + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# liquid +run-tests.sh -s liquid container + +c systemctl status liquidd +c elements-cli getpeerinfo +c elements-cli getnetworkinfo +c liquidswap-cli --help + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# tor +run-tests.sh container + +c cat /var/lib/tor/state +c ls -al /var/lib/tor/onion/ +c ls -al /var/lib/tor/onion/bitcoind +c ls -al /var/lib/tor/onion/clightning-rest + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# joinmarket +run-tests.sh -s joinmarket container + +c systemctl status joinmarket +c journalctl -u joinmarket + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# joinmarket-ob-watcher + +# This starts a container with WAN access, so that jm-ob-watcher +# can connect to the joinmarket IRC servers over Tor +run-tests.sh -s jm-ob-watcher container + +c systemctl status joinmarket-ob-watcher +c journalctl -u joinmarket-ob-watcher + +# Manually wait for string 'started http server, visit http://127.0.0.1:62601/' +# This can take >10 minutes when the Tor network is under heavy load. +# While connecting, errors like `We failed to connect and handshake with ANY directories...` +# may be shown. +c journalctl -f -u joinmarket-ob-watcher + +# Check webinterface +c curl localhost:62601 +nix run --inputs-from . nixpkgs#lynx -- --dump $ip:62601 +c curl -s localhost:62601 | grep -i "orders found" + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# rtl +# see ./topics/rtl.sh diff --git a/dev/dev-scenarios.nix b/dev/dev-scenarios.nix new file mode 100644 index 0000000..d809440 --- /dev/null +++ b/dev/dev-scenarios.nix @@ -0,0 +1,45 @@ +# Extra scenarios for developing and debugging + +{ lib, scenarios }: + +with lib; +{ + btcpayserver-regtest = { + imports = [ scenarios.regtestBase ]; + services.btcpayserver.enable = true; + test.container.exposeLocalhost = true; + # services.btcpayserver.lbtc = false; + }; + + # A node with internet access to test joinmarket-ob-watcher + jm-ob-watcher = { + services.joinmarket-ob-watcher.enable = true; + # Don't download blocks + services.bitcoind.extraConfig = '' + connect = 0; + ''; + test.container.exposeLocalhost = true; + test.container.enableWAN = true; + }; + + rtl-dev = { config, pkgs, lib, ... }: { + imports = [ + # scenarios.netnsBase + # scenarios.regtestBase + ]; + services.rtl = { + enable = true; + nodes.clightning = { + enable = true; + extraConfig.Settings.themeColor = "INDIGO"; + }; + # nodes.lnd.enable = false; + # services.rtl.nodes.reverseOrder = true; + nightTheme = true; + extraCurrency = "CHF"; + }; + test.container.exposeLocalhost = true; + nix-bitcoin.nodeinfo.enable = true; + # test.container.enableWAN = true; + }; +} diff --git a/dev/dev.sh b/dev/dev.sh new file mode 100644 index 0000000..125561c --- /dev/null +++ b/dev/dev.sh @@ -0,0 +1,128 @@ +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Access nix-bitcoin flake packages + +function nb() { + nix build --no-link --print-out-paths --print-build-logs "$@" +} + +# A package defined by nix-bitcoin +nb .#joinmarket +# Equivalent +nb .#modulesPkgs.joinmarket + +# A nix-bitcoin python package +nb .#nbPython3Packages.pyln-client + +# A pinned package from nixpkgs(-unstable) +nb .#pinned.electrs +# Equivalent +nb .#modulesPkgs.electrs + +## Eval packages +# Check version +nix eval .#joinmarket.version + +# Eval derivation. --raw is needed due to a Nix bug (https://github.com/NixOS/nix/issues/5731) +nix eval --raw .#joinmarket; echo + +# Check the version of a package in the nixpkgs(-unstable) inputs of the nix-bitcoin flake +nix eval --inputs-from . nixpkgs#electrs.version +nix eval --inputs-from . nixpkgs-unstable#electrs.version + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Inspect test systems + +# Build a test system +nix build -o /tmp/system --print-build-logs "$(nix eval --raw .#tests.default --apply ' + test: test.nodes.machine.system.build.toplevel.drvPath +')" +readlink /tmp/system +# Inspect system files +cat /tmp/system/activate +cat /tmp/system/etc/system/bitcoind.service + +# Evaluate a config value +nix eval .#tests.default --apply ' + test: test.nodes.machine.services.bitcoind.rpc.port +' + +# Evaluate a config value in a custom test +nix eval .#makeTest --apply ' + makeTest: let + config = (makeTest { + config = { + services.electrs.port = 10000; + }; + }).nodes.machine; + in + config.services.electrs.port +' + +# Evaluate a config value in a scenario defined in a file +nix eval --impure .#getTest --apply ' + getTest: let + config = (getTest { + name = "default"; + extraScenariosFile = builtins.getEnv("root") + "/scenarios.nix"; + }).nodes.machine; + in + config.services.bitcoind.port +' + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Manually run a nix-bitcoin container, without the test framework. +# This allows sharing directories with the container host via option `bindMounts.` + +read -rd '' src <<'EOF' || : + let + nix-bitcoin = builtins.getFlake "git+file://${toString ../.}"; + in + nix-bitcoin.inputs.extra-container.lib.buildContainers { + system = "x86_64-linux"; + inherit (nix-bitcoin.inputs) nixpkgs; + # legacyInstallDirs = true; + config = { + containers.nb-adhoc = { + # bindMounts."/shared" = { hostPath = "/my/hostpath"; isReadOnly = false; }; + extra.addressPrefix = "10.200.255"; + config = { + imports = [ nix-bitcoin.nixosModules.default ]; + services.bitcoind.enable = true; + nix-bitcoin.generateSecrets = true; + nix-bitcoin.nodeinfo.enable = true; + }; + }; + }; + } +EOF +nix run --impure --expr "$src" + +# Run command in container +nix shell --impure --expr "$src" -c container --run c nodeinfo +# TODO-EXTERNAL: Use this instead when https://github.com/NixOS/nix/issues/7444 is fixed +# nix run --impure --expr "$src" -- --run c nodeinfo + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Show build logs from a CI test run +# +# If a specific test derivation was already built successfully, the test is not rerun +# and the CI logs don't show the test output. + +# To view the test output: +# 1. Get the test store path at the end of the CI logs +# 2. +fetch_build_log() { + local log=$1 + nix cat-store --store https://nix-bitcoin.cachix.org "$log/output.xml" | + nix shell --inputs-from . nixpkgs#html-tidy -c tidy -xml -i - > /tmp/build-output.xml + echo + echo "Fetched log to /tmp/build-output.xml" +} +# Set this to your store path +fetch_build_log /nix/store/0cdjhvg84jsp47f3357812zjmj2wmz94-vm-test-run-nix-bitcoin-default + +# Show runtime +grep "script finished in" /tmp/build-output.xml + +# View XML with node folding +firefox /tmp/build-output.xml diff --git a/dev/topics/rtl.sh b/dev/topics/rtl.sh new file mode 100644 index 0000000..515f5eb --- /dev/null +++ b/dev/topics/rtl.sh @@ -0,0 +1,44 @@ +# shellcheck disable=SC2154 +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Docs +# https://github.com/Ride-The-Lightning/RTL +# config options: https://github.com/Ride-The-Lightning/RTL/blob/master/.github/docs/Application_configurations.md +# https://github.com/Ride-The-Lightning/c-lightning-REST +# local src: ~/s/RTL/ + +# Browse API docs (in container shell) +runuser -u "$(logname)" -- xdg-open "http://$ip:4001/api-docs/" + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Debug the service +run-tests.sh -s rtl-dev container + +c systemctl status rtl +c journalctl -u rtl +c cat /var/lib/rtl/RTL-Config.json + +c systemctl status clightning-rest + +# Open webinterface. Password: a +runuser -u "$(logname)" -- xdg-open "http://$ip:3000" + +#――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― +# Build RTL manually +# [[~/s/RTL/dockerfiles/Dockerfile][dockerfile]] +# [[~/s/RTL/package.json][package.json]] + +rtl_src=~/s/RTL +git clone https://github.com/Ride-The-Lightning/RTL "$rtl_src" + +nix build -o /tmp/nix-bitcoin-dev/nodejs --inputs-from . nixpkgs#nodejs-16_x +# Start a shell in a sandbox +env --chdir "$rtl_src" nix-bitcoin-firejail --whitelist="$rtl_src" --whitelist=/tmp/nix-bitcoin-dev/nodejs +PATH=/tmp/nix-bitcoin-dev/nodejs/bin:"$PATH" + +# Install +npm ci --omit=dev --omit=optional --no-update-notifier --ignore-scripts + +# Run +node rtl --help + +git clean -xdf # Cleanup repo diff --git a/docs/configuration.md b/docs/configuration.md index 878bf45..ef522a2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -135,7 +135,7 @@ You can use the same approach to allow connections to other services. ## Example: bitcoind ```shell -# 1. Stop bitcoind on your nodes +# 1. Stop bitcoind on your node ssh root@nix-bitcoin-node 'systemctl stop bitcoind' # Also stop bitcoind on the node that you'll be copying data from diff --git a/examples/README.md b/examples/README.md index 0ede270..6997ad9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -51,60 +51,7 @@ the node: ``` ### Tests -The internal test suite is also useful for exploring features.\ -The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on -the host system. - -`run-tests.sh` requires Nix >= 2.10. - -```bash -git clone https://github.com/fort-nix/nix-bitcoin -cd nix-bitcoin/test - -# Run a node in a VM. No tests are executed. -./run-tests.sh vm -systemctl status bitcoind - -# Run a Python test shell inside a VM node -./run-tests.sh debug -print(succeed("systemctl status bitcoind")) -run_test("bitcoind") - -# Run a node in a container. Requires systemd and root privileges. -./run-tests.sh container -c systemctl status bitcoind - -# Explore a single feature -./run-tests.sh --scenario electrs container - -# Run a command in a container -./run-tests.sh --scenario '{ - services.clightning.enable = true; - nix-bitcoin.nodeinfo.enable = true; -}' container --run c nodeinfo -``` -See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation. - -#### Flakes -Tests can also be directly accessed via Flakes: -```bash -# Build test -nix build --no-link ..#tests.default - -# Run a node in a VM. No tests are executed. -nix run ..#tests.default.vm - -# Run a Python test shell inside a VM node -nix run ..#tests.default.run -- --debug - -# Run a node in a container. Requires extra-container, systemd and root privileges -nix run ..#tests.default.container -nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05 - -# Run a command in a container -nix run ..#tests.default.container -- --run c nodeinfo -nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05 -``` +The [nix-bitcoin test suite](../test/README.md) is also useful for exploring features. ### Real-world example Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org diff --git a/flake.nix b/flake.nix index ece19d6..0c260e6 100644 --- a/flake.nix +++ b/flake.nix @@ -122,6 +122,8 @@ program = toString packages.runVM; }; }; + + devShells.default = import ./dev/dev-env/dev-shell.nix pkgs; } )); } diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..4cc4125 --- /dev/null +++ b/test/README.md @@ -0,0 +1,84 @@ +The [`run-tests.sh`](./run-tests.sh) command is most convenient and versatile way to run tests.\ +It leave no traces (outside of `/nix/store`) on the host system. + +`run-tests.sh` requires Nix >= 2.10. + +### Summary +```bash +./run-tests.sh [--scenario|-s ] [build|vm|debug|container] +``` + +See the top of [run-tests.sh](../test/run-tests.sh) for a complete documentation.\ +Test scenarios are defined in [tests.nix](./tests.nix) and [tests.py](tests.py). + +### Tutorial +#### Running tests +```bash +# Run the basic set of tests. These tests are also run on the GitHub CI server. +./run-tests.sh + +# Run the test for scenario `regtest`. +# The test is run via the Nix build system. Successful runs are cached. +./run-tests.sh -s regtest build +./run-tests.sh -s regtest # Shorthand, equivalent + +# To test a single service, use its name as a scenario. +./run-tests.sh -s clightning + +# When no scenario is specified, scenario `default` is used. +./run-tests.sh build +``` +#### Debugging +```bash +# Start a shell is inside a test VM. No tests are executed. +./run-tests.sh -s bitcoind vm +systemctl status bitcoind + +# Run a Python NixOS test shell inside a VM. +# See https://nixos.org/manual/nixos/stable/#ssec-machine-objects for available commands. +./run-tests.sh debug +print(succeed("systemctl status bitcoind")) +run_test("bitcoind") + +# Start a shell in a container node. Requires systemd and root privileges. +./run-tests.sh container + +# In the container shell: Run command in container (with prefix `c`) +c systemctl status bitcoind + +# Explore a single feature +./run-tests.sh -s electrs container + +# Run a command in a container. +# The container is deleted afterwards. +./run-tests.sh -s clightning container --run c lightning-cli getinfo + +# Define a custom scenario +./run-tests.sh --scenario '{ + services.clightning.enable = true; + nix-bitcoin.nodeinfo.enable = true; +}' container --run c nodeinfo +``` + +# Running tests with Flakes + +Tests can also be accessed via the nix-bitcoin flake: + +```bash +# Build test +nix build --no-link ..#tests.default + +# Run a node in a VM. No tests are executed. +nix run ..#tests.default.vm + +# Run a Python test shell inside a VM node +nix run ..#tests.default.run -- --debug + +# Run a node in a container. Requires extra-container, systemd and root privileges +nix run ..#tests.default.container +nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05 + +# Run a command in a container +nix run ..#tests.default.container -- --run c nodeinfo +nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05 +``` diff --git a/test/run-tests.sh b/test/run-tests.sh index 13eb647..57296de 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -4,7 +4,7 @@ # The tests (defined in ./tests.nix) use the NixOS testing framework and are executed in a VM. # # Usage: -# Run all tests +# Run the basic set of tests. These are also run on the CI server. # ./run-tests.sh # # Test specific scenario @@ -57,7 +57,8 @@ # and reading source files. # Files are copied to /tmp, a caching scheme helps minimizing copies. # -# To add custom scenarios, set the environment variable `scenarioOverridesFile`. +# Add custom scenarios from a file +# ./run-tests.sh --extra-scenarios ~/my/scenarios.nix ... set -eo pipefail @@ -66,6 +67,7 @@ scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd) args=("$@") scenario= outLinkPrefix= +scenarioOverridesFile= while :; do case $1 in --scenario|-s) @@ -88,6 +90,16 @@ while :; do exit 1 fi ;; + --extra-scenarios) + if [[ $2 ]]; then + scenarioOverridesFile=$2 + shift + shift + else + >&2 echo "Error: $1 requires an argument." + exit 1 + fi + ;; --copy-src|-c) shift if [[ ! $_nixBitcoinInCopiedSrc ]]; then @@ -113,7 +125,7 @@ makeTmpDir() { # Support explicit scenario definitions if [[ $scenario = *' '* ]]; then makeTmpDir - export scenarioOverridesFile=$tmpDir/scenario-overrides.nix + scenarioOverridesFile=$tmpDir/scenario-overrides.nix echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile" scenario=tmp fi @@ -195,7 +207,7 @@ buildTests() { nixInstantiateTest() { local attr=$1 shift - if [[ ${scenarioOverridesFile:-} ]]; then + if [[ $scenarioOverridesFile ]]; then local file="extraScenariosFile = \"$scenarioOverridesFile\";" else local file= diff --git a/test/tests.nix b/test/tests.nix index b0f9c69..cd6c824 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -53,12 +53,6 @@ let clboss.path = "${nbPkgs.clboss}/bin/clboss"; }; in map (plugin: pluginPkgs.${plugin}.path) enabled; - # Torified 'dig' subprocesses of clboss don't respond to SIGTERM and keep - # running for a long time when WAN is disabled, which prevents clightning units - # from stopping quickly. - # Set TimeoutStopSec for faster stopping. - systemd.services.clightning.serviceConfig.TimeoutStopSec = - mkIf config.services.clightning.plugins.clboss.enable "500ms"; tests.clightning-rest = cfg.clightning-rest.enable; @@ -90,6 +84,8 @@ let }; }; + nix-bitcoin.onionServices.lnd.public = true; + tests.lndconnect-onion-lnd = cfg.lnd.lndconnectOnion.enable; tests.lndconnect-onion-clightning = cfg.clightning-rest.lndconnectOnion.enable; @@ -97,7 +93,6 @@ let services.lightning-loop.certificate.extraIPs = [ "20.0.0.1" ]; tests.lightning-pool = cfg.lightning-pool.enable; - nix-bitcoin.onionServices.lnd.public = true; tests.charge-lnd = cfg.charge-lnd.enable; @@ -140,6 +135,13 @@ let # Avoid timeout failures on slow CI nodes systemd.services.postgresql.serviceConfig.TimeoutStartSec = "5min"; } + (mkIf config.services.clightning.plugins.clboss.enable { + # Torified 'dig' subprocesses of clboss don't respond to SIGTERM and keep + # running for a long time when WAN is disabled, which prevents clightning units + # from stopping quickly. + # Set TimeoutStopSec for faster stopping. + systemd.services.clightning.serviceConfig.TimeoutStopSec = "500ms"; + }) (mkIf config.test.features.clightningPlugins { services.clightning.plugins = { clboss.enable = true; @@ -313,7 +315,9 @@ let services.lnd.enable = true; services.bitcoind.prune = 1000; }; - }; + } // (import ../dev/dev-scenarios.nix { + inherit lib scenarios; + }); ## Example scenarios that showcase extra features exampleScenarios = with lib; { @@ -333,6 +337,31 @@ let # See ./lib/test-lib.nix for a description test.container.exposeLocalhost = true; }; + + ## Scenarios with a custom Python test + + # Variant 1: Define testing code that always runs + customTestSimple = { + networking.hostName = "myhost"; + + # Variant 1: Define testing code that always runs + test.extraTestScript = '' + succeed("[[ $(hostname) == myhost ]]") + ''; + }; + + # Variant 2: Define a test that can be enabled/disabled + # via the Nix module system. + customTestExtended = { + networking.hostName = "myhost"; + + tests.hostName = true; + test.extraTestScript = '' + @test("hostName") + def _(): + succeed("[[ $(hostname) == myhost ]]") + ''; + }; }; in { inherit scenarios;