joinmarket: 0.9.8 -> 0.9.9

fulcrum: 1.9.0 -> 1.9.1
lightning-loop: 0.20.0-beta -> 0.23.0-beta
lnd: 0.15.5-beta -> 0.16.2-beta

.cirrus.yml

@@ -9,7 +9,7 @@ task:
 
   container:
     # Defined in https://github.com/nix-community/docker-nixpkgs
-    image: nixpkgs/nix-flakes:nixos-22.05
+    image: nixpkgs/nix-flakes:nixos-22.11
 
   matrix:
     - name: modules_test
@@ -27,14 +27,16 @@ task:
           - scenario: default
           - scenario: netns
           - scenario: netnsRegtest
+          - scenario: trustedcoin
       # This script is run as root
       build_script:
         - echo "sandbox = true" >> /etc/nix/nix.conf
-        - nix shell --inputs-from . nixpkgs#{bash,coreutils,gawk,cachix} -c ./test/ci/build.sh
+        - nix shell --inputs-from . nixpkgs#{bash,coreutils,cachix} -c ./test/ci/build.sh $scenario
 
     - name: flake
       build_script:
         - nix flake check
+        - ./test/nixos-search/ci-test.sh
 
     - name: shellcheck
       build_script:

README.md

@@ -53,6 +53,8 @@ Hint: To show a table of contents, click the button (![Github TOC button](docs/i
 top left corner of the documents.
 
 <!-- TODO-EXTERNAL: -->
+<!-- Change query to `nix-bitcoin` when upstream search has been fixed -->
+* [NixOS options search](https://search.nixos.org/flakes?channel=unstable&sort=relevance&type=options&query=bitcoin)
 * [Hardware requirements](docs/hardware.md)
 * [Installation](docs/install.md)
 * [Configuration and maintenance](docs/configuration.md)
@@ -77,19 +79,22 @@ NixOS modules ([src](modules/modules.nix))
     * [prometheus](https://github.com/lightningd/plugins/tree/master/prometheus): lightning node exporter for the prometheus timeseries server
     * [rebalance](https://github.com/lightningd/plugins/tree/master/rebalance): keeps your channels balanced
     * [summary](https://github.com/lightningd/plugins/tree/master/summary): print a nice summary of the node status
+    * [trustedcoin](https://github.com/nbd-wtf/trustedcoin) [[experimental](docs/services.md#trustedcoin-hints)]: replaces bitcoind with trusted public explorers
     * [zmq](https://github.com/lightningd/plugins/tree/master/zmq): publishes notifications via ZeroMQ to configured endpoints
   * [clightning-rest](https://github.com/Ride-The-Lightning/c-lightning-REST): REST server for clightning
   * [lnd](https://github.com/lightningnetwork/lnd) with support for announcing an onion service and [static channel backups](https://github.com/lightningnetwork/lnd/blob/master/docs/recovery.md)
     * [Lightning Loop](https://github.com/lightninglabs/loop)
     * [Lightning Pool](https://github.com/lightninglabs/pool)
     * [charge-lnd](https://github.com/accumulator/charge-lnd): policy-based channel fee manager
-  * [lndconnect](https://github.com/LN-Zap/lndconnect): connect your wallet to lnd or clightning via a REST onion service
+  * [lndconnect](https://github.com/LN-Zap/lndconnect): connect your wallet to lnd or
+    clightning [via WireGuard](./docs/services.md#use-zeus-mobile-lightning-wallet-via-wireguard) or
+    [Tor](./docs/services.md#use-zeus-mobile-lightning-wallet-via-tor)
   * [Ride The Lightning](https://github.com/Ride-The-Lightning/RTL): web interface for `lnd` and `clightning`
   * [spark-wallet](https://github.com/shesek/spark-wallet)
-  * [electrs](https://github.com/romanz/electrs)
-  * [fulcrum](https://github.com/cculianu/Fulcrum) (see [the module](modules/fulcrum.nix) for a comparison to electrs)
+  * [electrs](https://github.com/romanz/electrs): Electrum server
+  * [fulcrum](https://github.com/cculianu/Fulcrum): Electrum server (see [the module](modules/fulcrum.nix) for a comparison with electrs)
   * [btcpayserver](https://github.com/btcpayserver/btcpayserver)
-  * [liquid](https://github.com/elementsproject/elements)
+  * [liquid](https://github.com/elementsproject/elements): federated sidechain
   * [JoinMarket](https://github.com/joinmarket-org/joinmarket-clientserver)
     * [JoinMarket Orderbook Watcher](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/orderbook.md)
   * [bitcoin-core-hwi](https://github.com/bitcoin-core/HWI)
@@ -97,7 +102,13 @@ NixOS modules ([src](modules/modules.nix))
   * [netns-isolation](modules/netns-isolation.nix): isolates applications on the network-level via network namespaces
   * [nodeinfo](modules/nodeinfo.nix): script which prints info about the node's services
   * [backups](modules/backups.nix): duplicity backups of all your node's important files
-  * [operator](modules/operator.nix): adds non-root user `operator` who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`)
+  * [operator](modules/operator.nix): configures a non-root user who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`)
+
+### Extension modules
+Extension modules are maintained in separate repositories and have their own review
+and release process.
+
+* [Mempool](https://github.com/fort-nix/nix-bitcoin-mempool): Bitcoin visualizer, explorer and API service
 
 Security
 ---
@@ -118,6 +129,10 @@ The nix-bitcoin security fund is a 2 of 3 bitcoin multisig address open for dona
 security researchers who discover vulnerabilities in nix-bitcoin or its upstream dependencies.\
 See [Security Fund](./SECURITY.md#nix-bitcoin-security-fund) for details.
 
+Developing
+---
+See [dev/README](./dev/README.md).
+
 Troubleshooting
 ---
 If you are having problems with nix-bitcoin check the [FAQ](docs/faq.md) or submit an issue.\

dev/README.md

@@ -0,0 +1,104 @@
+This directory contains docs and helper scripts for developing and debugging:
+
+- [`dev.sh`](./dev.sh): misc dev helpers
+- [`dev-features.sh`](./dev-features.sh): helpers for developing specific
+  nix-bitcoin features, like services
+- [`topics`](./topics) features specific topics
+- [`dev-scenarios.nix`](./dev-scenarios.nix): extra test scenarios used in the above scripts
+
+See also: [test/README.md](../test/README.md)
+
+## Run a dev shell
+
+There are two ways to run a dev shell:
+
+### 1. Run command `nix develop`
+
+This starts a shell with [`test/run-tests.sh`](../test/run-tests.sh) and
+the scripts in dir [`helper`](../helper) added to `PATH`.
+
+### 2. Setup and start the `direnv` dev env
+
+This is an opinionated, [direnv](https://direnv.net/)-based dev env, optimized for developer experience.
+
+[`dev-env/create.sh`](./dev-env/create.sh) creates a git repo with the following contents:
+- Dir `src` which contains the nix-bitcoin repo
+- Dir `bin` for helper scripts
+- File `scenarios.nix` for custom test scenarios
+- File `.envrc` that defines a [direnv](https://direnv.net/) environment,
+  mainly for adding nix-bitcoin and helper scripts to `PATH`
+
+#### Installation
+
+1. [Install direnv](https://direnv.net/docs/installation.html).\
+   If you use NixOS (and Bash as the default shell), just add the following to your system config:
+   ```nix
+     environment.systemPackages = [ pkgs.direnv ];
+     programs.bash.interactiveShellInit = ''
+       eval "$(direnv hook bash)"
+     '';
+   ```
+
+2. Create the dev env:
+   ```bash
+   # Set up a dev environment in dir ~/dev/nix-bitcoin.
+   # The dir is created automatically.
+   ./dev-env/create.sh ~/dev/nix-bitcoin
+
+   cd ~/dev/nix-bitcoin
+
+   # Enable direnv
+   direnv allow
+   ```
+
+3. Optional: Editor integration
+   - Add envrc support to your editor
+   - Setup your editor so you can easily execute lines or paragraphs from a shell script
+     file in a shell.\
+     This simplifies using dev helper scripts like [`./dev.sh`](./dev.sh).
+
+#### Explore the dev env
+```bash
+# The direnv is automatically activated when visiting any subdir of ~/dev/nix-bitcoin
+cd ~/dev/nix-bitcoin
+
+ls -al . bin lib
+
+# The direnv config file
+cat .envrc
+
+# You can use this file to define extra scenarios
+cat scenarios.nix
+
+# Binary `dev-run-tests` runs nix-bitcoin's `run-tests.sh` with extra scenarios from ./scenarios.nix
+# Example:
+# Run command `nodeinfo` in `myscenario` (defined in ./scenarios.nix) via a container
+dev-run-tests -s myscenario container --run c nodeinfo
+
+# Equivalent (shorthand)
+te -s myscenario container --run c nodeinfo
+
+# Run the tests for `myscenario` in a VM
+te -s myscenario
+
+# Start an interactive shell inside a VM
+te -s myscenario vm
+```
+
+See also: [test/README.md](../test/README.md)
+
+## Adding a new service
+
+It's easiest to use an existing service as a template:
+- [electrs.nix](../modules/electrs.nix): a basic service
+- [clightning.nix](../modules/clightning.nix): simple, but covers a few more features.\
+  (A `cli` binary and a runtime-composed config to include secrets.)
+- [rtl.nix](../modules/rtl.nix): includes a custom package, defined in [pkgs/rtl](../pkgs/rtl).\
+  Most other services use packages that are already included in nixpkgs.
+
+## Switching to a new NixOS release
+
+- [flake.nix](../flake.nix): update `nixpkgs.url`
+- [cirrus.yml](../.cirrus.yml): update toplevel container -> image attribute
+- [examples/configuration.nix](../examples/configuration.nix): update `system.stateVersion`
+- Treewide: check if any `TODO-EXTERNAL` comments can be resolved

dev/dev-env/create.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2016
+set -euo pipefail
+
+destDir=${1:-nix-bitcoin}
+
+scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+
+mkdir -p "$destDir/"{bin,lib}
+cd "$destDir"
+
+if [[ ! -e src ]]; then
+    echo "Cloning fort-nix/nix-bitcoin"
+    git clone https://github.com/fort-nix/nix-bitcoin src
+fi
+
+echo 'export root=$PWD
+export src=$root/src
+PATH_add bin
+PATH_add src/helper' > .envrc
+
+if [[ ! -e scenarios.nix ]]; then
+    cp "$scriptDir/template-scenarios.nix" scenarios.nix
+fi
+
+install -m 755 <(
+    echo '#!/usr/bin/env bash'
+    echo 'exec run-tests.sh --extra-scenarios "$root/scenarios.nix" "$@"'
+) bin/dev-run-tests
+
+install -m 755 <(
+    echo '#!/usr/bin/env bash'
+    echo 'exec $root/src/test/run-tests.sh --out-link-prefix /tmp/nix-bitcoin/test "$@"'
+) bin/run-tests.sh
+
+ln -sfn dev-run-tests bin/te
+
+## nix-bitcoin-firejail
+
+echo '# Add your shell config files here that should be accessible in the sandbox
+whitelist ${HOME}/.bashrc
+read-only ${HOME}/.bashrc' > lib/nix-bitcoin-firejail.conf
+
+install -m 755 <(
+    echo '#!/usr/bin/env bash'
+    echo '# A sandbox for running shells/binaries in an isolated environment:'
+    echo '# - The sandbox user is the calling user, with all capabilities dropped'
+    echo '#   and with no way to gain new privileges (e.g. via `sudo`).'
+    echo '# - $HOME is bind-mounted to a dir that only contains shell config files and files required by direnv.'
+    echo '#'
+    echo '# You can modify the firejail env by editing `lib/nix-bitcoin-firejail.conf` in your dev env dir.'
+    echo 'exec firejail --profile="$root/lib/nix-bitcoin-firejail.conf" --profile="$root/src/dev/dev-env/nix-bitcoin-firejail.conf" "$@"'
+) bin/nix-bitcoin-firejail
+
+echo "1" > lib/dev-env-version
+
+## git
+
+echo '/src' > .gitignore
+
+if [[ ! -e .git ]]; then
+    git init
+    git add .
+    git commit -a -m init
+fi

dev/dev-env/dev-shell.nix

@@ -0,0 +1,16 @@
+pkgs:
+
+pkgs.mkShell {
+  shellHook = ''
+    # A known rev from the master branch to test whether `nix develop`
+    # is called inside the nix-bitcoin repo
+    rev=5cafafd02777919c10e559b5686237fdefe920c2
+    if git cat-file -e $rev &>/dev/null; then
+      root=$(git rev-parse --show-toplevel)
+      export PATH=$root/test:$root/helper:$PATH
+    else
+      echo 'Error: `nix develop` must be called inside the nix-bitcoin repo.'
+      exit 1
+    fi
+  '';
+}

dev/dev-env/nix-bitcoin-firejail.conf

@@ -0,0 +1,25 @@
+include default.local
+include globals.local
+
+include disable-common.inc
+include disable-programs.inc
+
+caps.drop all
+netfilter
+noinput
+nonewprivs
+noroot
+notv
+novideo
+protocol unix,inet,inet6
+seccomp
+
+## Enable features
+
+allow-debuggers
+
+# Enable direnv configs
+whitelist ${HOME}/.config/direnv
+read-only ${HOME}/.config/direnv
+whitelist ${HOME}/.local/share/direnv
+read-only ${HOME}/.local/share/direnv

dev/dev-env/template-scenarios.nix

@@ -0,0 +1,20 @@
+{ pkgs, lib, scenarios, nix-bitcoin }:
+with lib;
+rec {
+  # For more examples, see `scenarios` and `exampleScenarios` in ./src/test/tests.nix
+
+  template = { config, pkgs, lib, ... }: {
+    imports = [
+      (nix-bitcoin + "/modules/presets/secure-node.nix")
+      scenarios.netnsBase
+      scenarios.regtestBase
+    ];
+    test.container.enableWAN = true;
+    test.container.exposeLocalhost = true;
+  };
+
+  myscenario = { config, pkgs, lib, ... }: {
+    services.clightning.enable = true;
+    nix-bitcoin.nodeinfo.enable = true;
+  };
+}

dev/dev-features.sh

@@ -0,0 +1,300 @@
+# shellcheck disable=SC2086,SC2154
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Run tests
+# See ../test/README.md for a tutorial
+# and ../test/run-tests.sh for a complete documentation
+
+# Start a shell in a container
+run-tests.sh -s electrs container
+
+# Run a command in a container.
+# The container is deleted afterwards.
+run-tests.sh -s electrs container --run c journalctl -u electrs
+
+# Run a bash command
+run-tests.sh -s bitcoind container --run c bash -c "sleep 1; journalctl -u bitcoind"
+
+run-tests.sh -s '{
+  imports = [ scenarios.regtestBase ];
+  services.electrs.enable = true;
+}' container --run c journalctl -u electrs
+
+run-tests.sh -s "{
+  services.electrs.enable = true;
+  nix-bitcoin.nodeinfo.enable = true;
+}" container --run c nodeinfo
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Get generic node infos
+
+# Start container shell
+run-tests.sh -s bitcoind container
+
+# Run commands inside the shell:
+
+# The node's services
+c systemctl status
+
+# Failed units
+c systemctl list-units --failed
+
+# Analyze container boot performance
+c systemd-analyze critical-chain
+
+# Listening TCP sockets
+c netstat -nltp
+# Listening sockets
+c netstat -nlp
+
+# The container root filesystem
+ls -al /var/lib/nixos-containers/nb-test
+
+# The container root filesystem on NixOS systems with stateVersion < 22.05
+ls -al /var/lib/containers/nb-test
+
+# Start a shell in the context of a service process.
+# Must be run inside the container (enter with cmd `c`).
+enter_service() {
+    name=$1
+    pid=$(systemctl show -p MainPID --value "$name")
+    IFS=- read -r uid gid  < <(stat -c "%u-%g" "/proc/$pid")
+    nsenter --all -t "$pid" --setuid "$uid" --setgid "$gid" bash
+}
+enter_service clightning
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# bitcoind
+run-tests.sh -s bitcoind container
+
+c systemctl status bitcoind
+c systemctl cat bitcoind
+c journalctl --output=short-precise -u bitcoind
+ls -al /var/lib/nixos-containers/nb-test/var/lib/bitcoind
+c bitcoin-cli getpeerinfo
+c bitcoin-cli getnetworkinfo
+c bitcoin-cli getblockchaininfo
+
+run-tests.sh -s '{
+  imports = [ scenarios.regtestBase ];
+  services.bitcoind.enable = true;
+}' container
+
+address=$(c bitcoin-cli getnewaddress)
+echo $address
+c bitcoin-cli generatetoaddress 10 $address
+
+# Run bitcoind with network access
+run-tests.sh -s "{
+  test.container.enableWAN = true;
+  services.bitcoind.enable = true;
+}" container --run c journalctl -u bitcoind -f
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# clightning
+run-tests.sh -s clightning container
+
+c systemctl status clightning
+c journalctl --output=short-precise -u clightning
+c lightning-cli getinfo
+
+# Plugins
+run-tests.sh -s "{
+  services.clightning.enable = true;
+  test.features.clightningPlugins = true;
+}" container
+
+c lightning-cli plugin list
+
+# Show plugin config
+nix eval --raw .#makeTest --apply '
+  makeTest: let
+   config = (makeTest {
+      config = {
+        services.clightning.enable = true;
+        test.features.clightningPlugins = true;
+      };
+    }).nodes.machine;
+  in
+    config.services.clightning.extraConfig
+'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# clightning-rest
+run-tests.sh -s clightning-rest container
+
+c systemctl status clightning-rest
+c journalctl -u clightning-rest
+c systemctl status clightning-rest-migrate-datadir
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# spark-wallet
+
+run-tests.sh -s "{
+  services.spark-wallet.enable = true;
+  test.container.exposeLocalhost = true;
+}" container
+
+c systemctl status spark-wallet
+c journalctl -u spark-wallet
+
+sparkAuth=$(c cat /secrets/spark-wallet-login | grep -ohP '(?<=login=).*')
+curl -v http://$sparkAuth@$ip:9737
+# Open in browser
+runuser -u "$(logname)" -- xdg-open http://$sparkAuth@$ip:9737
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# electrs
+
+run-tests.sh -s "{
+  imports = [ scenarios.regtestBase ];
+  services.electrs.enable = true;
+}" container
+
+c systemctl status electrs
+c systemctl cat electrs
+c journalctl --output=short-precise -u electrs
+
+electrs_rpc() {
+  echo "$1" | c nc 127.0.0.1 50001 | head -1 | jq
+}
+electrs_rpc '{"method": "server.version", "id": 0, "params": ["electrum/3.3.8", "1.4"]}'
+electrs_rpc '{"method": "blockchain.headers.subscribe", "id": 0}'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# fulcrum
+
+run-tests.sh -s "{
+  imports = [ scenarios.regtestBase ];
+  services.fulcrum.enable = true;
+}" container
+
+c systemctl status fulcrum
+c systemctl cat fulcrum
+c journalctl --output=short-precise -u fulcrum
+
+fulcrum_rpc() {
+    echo "$1" | c nc 127.0.0.1 50002 | head -1 | jq
+}
+fulcrum_rpc '{"method": "server.version", "id": 0, "params": ["electrum/3.3.8", "1.4"]}'
+fulcrum_rpc '{"method": "blockchain.headers.subscribe", "id": 0}'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# lnd
+run-tests.sh -s lnd container
+c systemctl status lnd
+c journalctl -u lnd
+c lncli getinfo
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# lightning-loop
+run-tests.sh -s lightning-loop container
+c systemctl status lightning-loop
+c journalctl -u lightning-loop
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# btcpayserver
+# https://docs.btcpayserver.org/Development/GreenFieldExample/
+
+run-tests.sh -s btcpayserver-regtest container
+
+c systemctl status btcpayserver
+c journalctl -u btcpayserver
+c systemctl cat btcpayserver
+
+c systemctl status nbxplorer
+c journalctl -u nbxplorer
+
+## Access the API
+request() {
+    local type=$1
+    local method=$2
+    local body=$3
+    shift; shift; shift
+    curl -sSL -H "Content-Type: application/json" -X $type --user "a@a.a:aaaaaa" \
+         -d "$body" "$@" "$ip:23000/api/v1/$method" | jq
+}
+post() {
+    local method=$1
+    local body=$2
+    shift; shift
+    request post "$method" "$body" "$@"
+}
+get() {
+    local method=$1
+    request get "$method"
+}
+
+# Create new user
+post users '{"email": "a@a.a", "password": "aaaaaa", "isAdministrator": true}'
+
+# Login with:
+# user: a@a.a
+# password: aaaaaa
+runuser -u "$(logname)" -- xdg-open http://$ip:23000
+
+# create store
+post stores '{"name": "a", "defaultPaymentMethod": "BTC_LightningNetwork"}'
+post stores '{"name": "a", "defaultPaymentMethod": "BTC"}'
+
+store=$(get stores | jq -r .[].id)
+echo $store
+get stores/$store
+get stores/$store/payment-methods
+get stores/$store/payment-methods/LightningNetwork
+
+# Connect to internal lightning node (internal API, doesn't work)
+# Lightning must be manually setup via the webinterface.
+post stores/$store/lightning/BTC/setup "" --data-raw 'LightningNodeType=Internal&ConnectionString=&command=save'
+
+nix run --inputs-from . nixpkgs#lynx -- --dump http://$ip:23000/embed/$store/BTC/ln
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# liquid
+run-tests.sh -s liquid container
+
+c systemctl status liquidd
+c elements-cli getpeerinfo
+c elements-cli getnetworkinfo
+c liquidswap-cli --help
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# tor
+run-tests.sh container
+
+c cat /var/lib/tor/state
+c ls -al /var/lib/tor/onion/
+c ls -al /var/lib/tor/onion/bitcoind
+c ls -al /var/lib/tor/onion/clightning-rest
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# joinmarket
+run-tests.sh -s joinmarket container
+
+c systemctl status joinmarket
+c journalctl -u joinmarket
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# joinmarket-ob-watcher
+
+# This starts a container with WAN access, so that jm-ob-watcher
+# can connect to the joinmarket IRC servers over Tor
+run-tests.sh -s jm-ob-watcher container
+
+c systemctl status joinmarket-ob-watcher
+c journalctl -u joinmarket-ob-watcher
+
+# Manually wait for string 'started http server, visit http://127.0.0.1:62601/'
+# This can take >10 minutes when the Tor network is under heavy load.
+# While connecting, errors like `We failed to connect and handshake with ANY directories...`
+# may be shown.
+c journalctl -f -u joinmarket-ob-watcher
+
+# Check webinterface
+c curl localhost:62601
+nix run --inputs-from . nixpkgs#lynx -- --dump $ip:62601
+c curl -s localhost:62601 | grep -i "orders found"
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# rtl
+# see ./topics/rtl.sh

dev/dev-scenarios.nix

@@ -0,0 +1,78 @@
+# Extra scenarios for developing and debugging
+
+{ lib, scenarios }:
+
+with lib;
+{
+  btcpayserver-regtest = {
+    imports = [ scenarios.regtestBase ];
+    services.btcpayserver.enable = true;
+    test.container.exposeLocalhost = true;
+    # services.btcpayserver.lbtc = false;
+
+    # Required for testing interactive plugin installation
+    test.container.enableWAN = true;
+  };
+
+  # A node with internet access to test joinmarket-ob-watcher
+  jm-ob-watcher = {
+    services.joinmarket-ob-watcher.enable = true;
+    # Don't download blocks
+    services.bitcoind.extraConfig = ''
+      connect = 0;
+    '';
+    test.container.exposeLocalhost = true;
+    test.container.enableWAN = true;
+  };
+
+  rtl-dev = { config, pkgs, lib, ... }: {
+    imports = [
+      # scenarios.netnsBase
+      # scenarios.regtestBase
+    ];
+    services.rtl = {
+      enable = true;
+      nodes.clightning = {
+        enable = true;
+        extraConfig.Settings.themeColor = "INDIGO";
+      };
+      # nodes.lnd.enable = false;
+      # services.rtl.nodes.reverseOrder = true;
+      nightTheme = true;
+      extraCurrency = "CHF";
+    };
+    test.container.exposeLocalhost = true;
+    nix-bitcoin.nodeinfo.enable = true;
+    # test.container.enableWAN = true;
+  };
+
+  wireguard-lndconnect-online = { config, pkgs, lib, ... }: {
+    imports = [
+      ../modules/presets/wireguard.nix
+      scenarios.regtestBase
+    ];
+
+    # 51820 (default wg port) + 1
+    networking.wireguard.interfaces.wg-nb.listenPort = 51821;
+    test.container.enableWAN = true;
+    # test.container.exposeLocalhost = true;
+
+    services.clightning.extraConfig = "disable-dns";
+
+    services.lnd = {
+      enable = true;
+      lndconnect = {
+        enable = true;
+        onion = true;
+      };
+    };
+    services.clightning-rest = {
+      enable = true;
+      lndconnect = {
+        enable = true;
+        onion = true;
+      };
+    };
+    nix-bitcoin.nodeinfo.enable = true;
+  };
+}

dev/dev.sh

@@ -0,0 +1,128 @@
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Access nix-bitcoin flake packages
+
+function nb() {
+    nix build --no-link --print-out-paths --print-build-logs "$@"
+}
+
+# A package defined by nix-bitcoin
+nb .#joinmarket
+# Equivalent
+nb .#modulesPkgs.joinmarket
+
+# A nix-bitcoin python package
+nb .#nbPython3Packages.pyln-client
+
+# A pinned package from nixpkgs(-unstable)
+nb .#pinned.electrs
+# Equivalent
+nb .#modulesPkgs.electrs
+
+## Eval packages
+# Check version
+nix eval .#joinmarket.version
+
+# Eval derivation. --raw is needed due to a Nix bug (https://github.com/NixOS/nix/issues/5731)
+nix eval --raw .#joinmarket; echo
+
+# Check the version of a package in the nixpkgs(-unstable) inputs of the nix-bitcoin flake
+nix eval --inputs-from . nixpkgs#electrs.version
+nix eval --inputs-from . nixpkgs-unstable#electrs.version
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Inspect test systems
+
+# Build a test system
+nix build -o /tmp/system --print-build-logs "$(nix eval --raw .#tests.default --apply '
+  test: test.nodes.machine.system.build.toplevel.drvPath
+')"
+readlink /tmp/system
+# Inspect system files
+cat /tmp/system/activate
+cat /tmp/system/etc/system/bitcoind.service
+
+# Evaluate a config value
+nix eval .#tests.default --apply '
+  test: test.nodes.machine.services.bitcoind.rpc.port
+'
+
+# Evaluate a config value in a custom test
+nix eval .#makeTest --apply '
+  makeTest: let
+   config = (makeTest {
+      config = {
+        services.electrs.port = 10000;
+      };
+    }).nodes.machine;
+  in
+    config.services.electrs.port
+'
+
+# Evaluate a config value in a scenario defined in a file
+nix eval --impure .#getTest --apply '
+  getTest: let
+    config = (getTest {
+      name = "default";
+      extraScenariosFile = builtins.getEnv("root") + "/scenarios.nix";
+    }).nodes.machine;
+  in
+    config.services.bitcoind.port
+'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Manually run a nix-bitcoin container, without the test framework.
+# This allows sharing directories with the container host via option `bindMounts.`
+
+read -rd '' src <<'EOF' || :
+  let
+    nix-bitcoin = builtins.getFlake "git+file://${toString ../.}";
+  in
+    nix-bitcoin.inputs.extra-container.lib.buildContainers {
+      system = "x86_64-linux";
+      inherit (nix-bitcoin.inputs) nixpkgs;
+      # legacyInstallDirs = true;
+      config = {
+        containers.nb-adhoc = {
+          # bindMounts."/shared" = { hostPath = "/my/hostpath"; isReadOnly = false; };
+          extra.addressPrefix = "10.200.255";
+          config = {
+            imports = [ nix-bitcoin.nixosModules.default ];
+            services.bitcoind.enable = true;
+            nix-bitcoin.generateSecrets = true;
+            nix-bitcoin.nodeinfo.enable = true;
+          };
+        };
+      };
+    }
+EOF
+nix run --impure --expr "$src"
+
+# Run command in container
+nix shell --impure --expr "$src" -c container --run c nodeinfo
+# TODO-EXTERNAL: Use this instead when https://github.com/NixOS/nix/issues/7444 is fixed
+# nix run --impure --expr "$src" -- --run c nodeinfo
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Show build logs from a CI test run
+#
+# If a specific test derivation was already built successfully, the test is not rerun
+# and the CI logs don't show the test output.
+
+# To view the test output:
+# 1. Get the test store path at the end of the CI logs
+# 2.
+fetch_build_log() {
+    local log=$1
+    nix cat-store --store https://nix-bitcoin.cachix.org "$log/output.xml" |
+        nix shell --inputs-from . nixpkgs#html-tidy -c tidy -xml -i - > /tmp/build-output.xml
+    echo
+    echo "Fetched log to /tmp/build-output.xml"
+}
+# Set this to your store path
+fetch_build_log /nix/store/0cdjhvg84jsp47f3357812zjmj2wmz94-vm-test-run-nix-bitcoin-default
+
+# Show runtime
+grep "script finished in" /tmp/build-output.xml
+
+# View XML with node folding
+firefox /tmp/build-output.xml

dev/topics/lndconnect-and-wireguard.sh

@@ -0,0 +1,64 @@
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Test Tor and WireGuard connections on a mobile device
+
+# 1. Run container
+run-tests.sh -s wireguard-lndconnect-online container
+
+# 2. Test connecting via Tor
+# Print QR codes for lnd, clightning-rest connections via Tor
+c lndconnect
+c lndconnect-clightning
+# Add these to Zeus >= 0.7.1.
+# To explicitly check if the connection is successful, press the node logo in the top
+# left corner, and then "Node Info".
+
+# Debug
+c lndconnect --url
+c lndconnect-clightning --url
+
+# 3. Test connecting via WireGuard
+
+# 3.1 Forward WireGuard port from the container host to the container
+iptables -t nat -A PREROUTING -p udp --dport 51821 -j DNAT --to-destination 10.225.255.2
+
+# 3.2. Optional: When your container host has an external firewall,
+# forward the WireGuard port to the container host:
+# - Port: 51821
+# - Protocol: UDP
+# - Destination: IPv4 of the container host
+
+# 3.2 Print QR code and setup wireguard on the mobile device
+c nix-bitcoin-wg-connect
+c nix-bitcoin-wg-connect --text
+
+# Print QR codes for lnd, clightning-rest connections via WireGuard
+c lndconnect-wg
+c lndconnect-clightning-wg
+# Add these to Zeus >= 0.7.1.
+# To explicitly check if the connection is successful, press the node logo in the top
+# left corner, and then "Node Info".
+
+# Debug
+c lndconnect-wg --url
+c lndconnect-clightning-wg --url
+
+# 3.3.remove external firewall port forward, remove local port forward:
+iptables -t nat -D PREROUTING -p udp --dport 51821 -j DNAT --to-destination 10.225.255.2
+# Now exit the container shell
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Debug lndconnect
+
+run-tests.sh -s wireguard-lndconnect-online container
+
+c nodeinfo
+
+c lndconnect --url
+c lndconnect-wg --url
+c lndconnect-clightning --url
+c lndconnect-clightning-wg --url
+
+c lndconnect
+c lndconnect-wg
+c lndconnect-clightning
+c lndconnect-clightning-wg

dev/topics/rtl.sh

@@ -0,0 +1,47 @@
+# shellcheck disable=SC2154
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Docs
+# https://github.com/Ride-The-Lightning/RTL
+# config options: https://github.com/Ride-The-Lightning/RTL/blob/master/.github/docs/Application_configurations.md
+# https://github.com/Ride-The-Lightning/c-lightning-REST
+# local src: ~/s/RTL/
+
+# Browse API docs (in container shell)
+runuser -u "$(logname)" -- xdg-open "http://$ip:4001/api-docs/"
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Debug the service
+run-tests.sh -s rtl-dev container
+
+c systemctl status rtl
+c journalctl -u rtl
+c cat /var/lib/rtl/RTL-Config.json
+
+c systemctl status clightning-rest
+
+# Open webinterface. Password: a
+runuser -u "$(logname)" -- xdg-open "http://$ip:3000"
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Build RTL manually
+# [[~/s/RTL/dockerfiles/Dockerfile][dockerfile]]
+# [[~/s/RTL/package.json][package.json]]
+
+rtl_src=~/s/RTL
+git clone https://github.com/Ride-The-Lightning/RTL "$rtl_src"
+
+nix build -o /tmp/nix-bitcoin-dev/nodejs --inputs-from . nixpkgs#nodejs-16_x
+# Start a shell in a sandbox
+env --chdir "$rtl_src" nix-bitcoin-firejail --whitelist="$rtl_src" --whitelist=/tmp/nix-bitcoin-dev/nodejs
+PATH=/tmp/nix-bitcoin-dev/nodejs/bin:"$PATH"
+
+# Install
+npm ci --omit=dev --omit=optional --no-update-notifier --ignore-scripts
+# If the above fails, try: (details: https://github.com/Ride-The-Lightning/RTL/issues/1182)
+npm ci --omit=dev --omit=optional --no-update-notifier --ignore-scripts --legacy-peer-deps
+
+
+# Run
+node rtl --help
+
+git clean -xdf # Cleanup repo

docs/configuration.md

@@ -135,7 +135,7 @@ You can use the same approach to allow connections to other services.
 ## Example: bitcoind
 
 ```shell
-# 1. Stop bitcoind on your nodes
+# 1. Stop bitcoind on your node
 ssh root@nix-bitcoin-node 'systemctl stop bitcoind'
 # Also stop bitcoind on the node that you'll be copying data from
 

docs/install.md

@@ -46,7 +46,7 @@ This is borrowed from the [NixOS manual](https://nixos.org/nixos/manual/index.ht
     ls /sys/firmware/efi
     ```
 
-    If the file exists exists, you should continue the installation for UEFI otherwise for Legacy Boot.
+    If the file exists you should continue the installation for UEFI otherwise for Legacy Boot.
 
 4. Option 1: Partition and format for UEFI
 

docs/services.md

@@ -142,60 +142,154 @@ You can find the `<onion-address>` with command `nodeinfo`.
 The default password location is `$secretsDir/rtl-password`.
 See: [Secrets dir](./configuration.md#secrets-dir)
 
-# Use LND or clightning with Zeus (mobile wallet) via Tor
-1. Install [Zeus](https://zeusln.app)
+# Use Zeus (mobile lightning wallet) via Tor
+1. Install [Zeus](https://zeusln.app) (version ≥ 0.7.1)
 
 2. Edit your `configuration.nix`
 
    ##### For lnd
 
    Add the following config:
-   ```
-   services.lnd.lndconnectOnion.enable = true;
+   ```nix
+   services.lnd.lndconnect = {
+     enable = true;
+     onion = true;
+   };
    ```
 
    ##### For clightning
 
    Add the following config:
-   ```
+   ```nix
    services.clightning-rest = {
      enable = true;
-     lndconnectOnion.enable = true;
+     lndconnect = {
+       enable = true;
+       onion = true;
+     };
    };
    ```
 
 3. Deploy your configuration
 
-3. Run the following command on your node (as user `operator`) to create a QR code
+4. Run the following command on your node (as user `operator`) to create a QR code
    with address and authentication information:
 
    ##### For lnd
    ```
-   lndconnect-onion
+   lndconnect
    ```
 
    ##### For clightning
    ```
-   lndconnect-onion-clightning
+   lndconnect-clightning
    ```
 
-4. Configure Zeus
-   - Add a new node
-   - Select `Scan lndconnect config` (at the bottom) and scan the QR code
-   - For clightning: Set `Node interface` to `c-lightning-REST`
+5. Configure Zeus
+   - Add a new node and scan the QR code
    - Click `Save node config`
    - Start sending and stacking sats privately
 
 ### Additional lndconnect features
-Create plain text URLs or QR code images:
+- Create a plain text URL:
+  ```bash
+  lndconnect --url
   ```
-lndconnect-onion --url
-lndconnect-onion --image
+- Set a custom host. By default, `lndconnect` detects the system's external IP and uses it as the host.
+  ```bash
+  lndconnect --host myhost
+  ```
+
+# Use Zeus (mobile lightning wallet) via WireGuard
+
+Connecting Zeus directly to your node is much faster than using Tor, but a bit more complex to setup.
+
+There are two ways to establish a secure, direct connection:
+
+- Connecting via TLS. This requires installing your lightning app's
+  TLS Certificate on your mobile device.
+
+- Connecting via WireGuard. This approach is simpler and more versatile, and is
+  described in this guide.
+
+1. Install [Zeus](https://zeusln.app) (version ≥ 0.7.1) and
+   [WireGuard](https://www.wireguard.com/install/) on your mobile device.
+
+2. Add the following to your `configuration.nix`:
+   ```nix
+   imports = [
+     # Use this line when using the default deployment method
+     <nix-bitcoin/modules/presets/wireguard.nix>
+
+     # Use this line when using Flakes
+     (nix-bitcoin + /modules/presets/wireguard.nix)
+   ]
+
+   # For lnd
+   services.lnd.lndconnect.enable = true;
+
+   # For clightning
+   services.clightning-rest = {
+     enable = true;
+     lndconnect.enable = true;
+   };
+   ```
+3. Deploy your configuration.
+
+4. If your node is behind an external firewall or NAT, add the following port forwarding
+   rule to the external device:
+   - Port: 51820 (the default value of option `networking.wireguard.interfaces.wg-nb.listenPort`)
+   - Protocol: UDP
+   - Destination: IP of your node
+
+5. Setup WireGuard on your mobile device.
+
+   Run the following command on your node (as user `operator`) to create a QR code
+   for WireGuard:
+   ```bash
+   nix-bitcoin-wg-connect
+
+   # For debugging: Show the WireGuard config as text
+   nix-bitcoin-wg-connect --text
+   ```
+   The above commands automatically detect your node's external IP.\
+   To set a custom IP or hostname, run the following:
+   ```
+   nix-bitcoin-wg-connect 93.184.216.34
+   nix-bitcoin-wg-connect mynode.org
+   ```
+
+   Configure WireGuard:
+   - Press the `+` button in the bottom right corner
+   - Scan the QR code
+   - Add the tunnel
+
+6. Setup Zeus
+
+   Run the following command on your node (as user `operator`) to create a QR code for Zeus:
+
+   ##### For lnd
+   ```
+   lndconnect-wg
+   ```
+
+   ##### For clightning
+   ```
+   lndconnect-clightning-wg
+   ```
+
+   Configure Zeus:
+   - Add a new node and scan the QR code
+   - Click `Save node config`
+   - On the certificate warning screen, click `I understand, save node config`.\
+     Certificates are not needed when connecting via WireGuard.
+   - Start sending and stacking sats privately
+
+### Additional lndconnect features
+Create a plain text URL:
+```bash
+lndconnect-wg --url
 ``````
-Create a QR code for a custom hostname:
-```
-lndconnect-onion --host=mynode.org
-```
 
 # Connect to spark-wallet
 ### Requirements
@@ -527,3 +621,27 @@ services.clightning = {
 ```
 
 Please have a look at the module for a plugin (e.g. [prometheus.nix](../modules/clightning-plugins/prometheus.nix)) to learn its configuration options.
+
+### Trustedcoin hints
+The [trustedcoin](https://github.com/nbd-wtf/trustedcoin) plugin use a Tor
+proxy for all of its external connections by default. That's why you can
+sometimes face issues with your connections to esploras getting blocked.
+
+An example of clightning log error output in a case your connections are getting blocked:
+
+```
+lightningd[5138]: plugin-trustedcoin estimatefees error: https://blockstream.info/api error: 403 Forbidden
+```
+
+```
+lightningd[4933]: plugin-trustedcoin getblock error: got something that isn't a block hash: <html><head>
+lightningd[4933]: <meta http-equiv="content-type" content="text/html;
+```
+
+If you face these issues and you still need to use trustedcoin, use can disable
+clightning's tor hardening by setting this option in your `configuration.nix`
+file:
+
+```
+services.clightning.tor.enforce = false;
+```

examples/README.md

@@ -51,41 +51,11 @@ the node:
 ```
 
 ### Tests
-The internal test suite is also useful for exploring features.\
-The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on
-the host system.
-
-```bash
-git clone https://github.com/fort-nix/nix-bitcoin
-cd nix-bitcoin/test
-
-# Run a node in a VM. No tests are executed.
-./run-tests.sh vm
-systemctl status bitcoind
-
-# Run a Python test shell inside a VM node
-./run-tests.sh debug
-print(succeed("systemctl status bitcoind"))
-run_test("bitcoind")
-
-# Run a node in a container. Requires systemd and root privileges.
-./run-tests.sh container
-c systemctl status bitcoind
-
-# Explore a single feature
-./run-tests.sh --scenario electrs container
-
-# Run a command in a container
-./run-tests.sh --scenario '{
-  services.clightning.enable = true;
-  nix-bitcoin.nodeinfo.enable = true;
-}' container --run c nodeinfo
-```
-See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.
+The [nix-bitcoin test suite](../test/README.md) is also useful for exploring features.
 
 ### Real-world example
 Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org
-to see the configuration of a nix-bitcoin node that's used in production.
+to see the configuration of a Flakes-based nix-bitcoin node that's used in production.
 
 The commands in `shell.nix` allow you to locally run the node in a VM or container.
 

examples/configuration.nix

@@ -56,13 +56,18 @@
   #
   # == REST server
   # Set this to create a clightning REST onion service.
-  # This also adds binary `lndconnect-onion-clightning` to the system environment.
+  # This also adds binary `lndconnect-clightning` to the system environment.
   # This binary creates QR codes or URLs for connecting applications to clightning
-  # via the REST onion service (see ../docs/services.md).
+  # via the REST onion service.
+  # You can also connect via WireGuard instead of Tor.
+  # See ../docs/services.md for details.
   #
   # services.clightning-rest = {
   #   enable = true;
-  #   lndconnectOnion.enable = true;
+  #   lndconnect = {
+  #     enable = true;
+  #     onion = true;
+  #   };
   # };
 
   ### LND
@@ -78,11 +83,17 @@
   # The onion service is automatically announced to peers.
   # nix-bitcoin.onionServices.lnd.public = true;
   #
-  # Set this to create an lnd REST onion service.
-  # This also adds binary `lndconnect-onion` to the system environment.
+  # Set this to create a lnd REST onion service.
+  # This also adds binary `lndconnect` to the system environment.
   # This binary generates QR codes or URLs for connecting applications to lnd via the
-  # REST onion service (see ../docs/services.md).
-  # services.lnd.lndconnectOnion.enable = true;
+  # REST onion service.
+  # You can also connect via WireGuard instead of Tor.
+  # See ../docs/services.md for details.
+  #
+  # services.lnd.lndconnect = {
+  #   enable = true;
+  #   onion = true;
+  # };
   #
   ## WARNING
   # If you use lnd, you should manually backup your wallet mnemonic
@@ -287,10 +298,10 @@
   # this value at the release version of the first install of this system.
   # Before changing this value read the documentation for this option
   # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "22.05"; # Did you read the comment?
+  system.stateVersion = "22.11"; # Did you read the comment?
 
   # The nix-bitcoin release version that your config is compatible with.
   # When upgrading to a backwards-incompatible release, nix-bitcoin will display an
   # an error and provide instructions for migrating your config to the new release.
-  nix-bitcoin.configVersion = "0.0.70";
+  nix-bitcoin.configVersion = "0.0.85";
 }

examples/deploy-container-minimal.sh

@@ -1,17 +1,11 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then
-    echo "Running script in nix shell env..."
-    cd "${BASH_SOURCE[0]%/*}"
-    exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*"
-else
-    cd "$NIX_BITCOIN_EXAMPLES_DIR"
-fi
-
 tmpDir=$(mktemp -d /tmp/nix-bitcoin-minimal-container.XXX)
 trap 'rm -rf $tmpDir' EXIT
 
+cd "${BASH_SOURCE[0]%/*}"
+
 # Modify importable-configuration.nix to use the local <nix-bitcoin>
 # source instead of fetchTarball
 <importable-configuration.nix sed '
@@ -31,4 +25,4 @@ cat > "$tmpDir/configuration.nix" <<EOF
   }
 EOF
 
-"${BASH_SOURCE[0]%/*}/deploy-container.sh" "$tmpDir/configuration.nix" "$@"
+./deploy-container.sh "$tmpDir/configuration.nix" "$@"

examples/deploy-container.sh

@@ -8,23 +8,21 @@ set -euo pipefail
 # Run with option `--interactive` or `-i` to start a shell for interacting with
 # the node.
 
-if [[ ! -v NIX_BITCOIN_EXAMPLES_DIR ]]; then
-    echo "Running script in nix shell env..."
-    cd "${BASH_SOURCE[0]%/*}"
-    exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*"
-else
-    cd "$NIX_BITCOIN_EXAMPLES_DIR"
+if [[ $EUID != 0 ]]; then
+    # NixOS containers require root permissions
+    exec sudo "${BASH_SOURCE[0]}" "$@"
 fi
 
-if [[ $(sysctl -n net.ipv4.ip_forward || sudo sysctl -n net.ipv4.ip_forward) != 1 ]]; then
+if [[ $(sysctl -n net.ipv4.ip_forward) != 1 ]]; then
     echo "Error: IP forwarding (net.ipv4.ip_forward) is not enabled."
     echo "Needed for container WAN access."
     exit 1
 fi
 
-if [[ $EUID != 0 ]]; then
-    # NixOS containers require root permissions
-    exec sudo "PATH=$PATH" "NIX_PATH=$NIX_PATH" "NIX_BITCOIN_EXAMPLES_DIR=$NIX_BITCOIN_EXAMPLES_DIR" "${BASH_SOURCE[0]}" "$@"
+if [[ ! -v DEPLOY_CONTAINER_NIX_SHELL ]]; then
+    echo "Running script in nix shell env..."
+    cd "${BASH_SOURCE[0]%/*}"
+    DEPLOY_CONTAINER_NIX_SHELL=1 exec nix-shell --run "./${BASH_SOURCE[0]##*/} $*"
 fi
 
 interactive=

examples/flakes/flake.nix

@@ -14,7 +14,7 @@
         # Import the secure-node preset, an opinionated config to enhance security
         # and privacy.
         #
-        # "${nix-bitcoin}/modules/presets/secure-node.nix"
+        # (nix-bitcoin + "/modules/presets/secure-node.nix")
 
         {
           # Automatically generate all secrets required by services.

examples/qemu-vm/minimal-vm.nix

@@ -13,13 +13,13 @@ rec {
     QEMU_OPTS="-smp $(nproc) -m 1500" ${vm}/bin/run-*-vm
   '';
 
-  vm = (import "${nixpkgs}/nixos" {
+  vm = (import (nixpkgs + "/nixos") {
     inherit system;
     configuration = { config, lib, modulesPath, ... }: {
       imports = [
         nix-bitcoin.nixosModules.default
-        "${nix-bitcoin}/modules/presets/secure-node.nix"
-        "${modulesPath}/virtualisation/qemu-vm.nix"
+        (nix-bitcoin + "/modules/presets/secure-node.nix")
+        (modulesPath + "/virtualisation/qemu-vm.nix")
       ];
 
       virtualisation.graphics = false;
@@ -29,6 +29,9 @@ rec {
       # For faster startup in offline VMs
       services.clightning.extraConfig = "disable-dns";
 
+      # Avoid lengthy build of the nixos manual
+      documentation.nixos.enable = false;
+
       nixpkgs.pkgs = pkgs;
       services.getty.autologinUser = "root";
       nix.nixPath = [ "nixpkgs=${nixpkgs}" ];
@@ -40,6 +43,7 @@ rec {
         - nodeinfo
         - systemctl status bitcoind
         - systemctl status clightning
+        - lightning-cli getinfo
       '';
 
       # Power off VM when the user exits the shell

examples/qemu-vm/vm-config.nix

@@ -3,7 +3,7 @@
   # Disable the hardened preset to improve VM performance
   disabledModules = [ <nix-bitcoin/modules/presets/hardened.nix> ];
 
-  imports = [ "${modulesPath}/virtualisation/qemu-vm.nix" ];
+  imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ];
 
   config = {
     virtualisation.graphics = false;

flake.lock

@@ -1,12 +1,38 @@
 {
   "nodes": {
-    "flake-utils": {
+    "extra-container": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1659877975,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "lastModified": 1679648217,
+        "narHash": "sha256-aq2J5Hj5IE8X8X/7v3n0wcv8n+FLzzENbcCF9xqhxAc=",
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "rev": "40c73f5e3292e73d6ce91625d9751be84fde17cb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "erikarvstedt",
+        "repo": "extra-container",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
         "type": "github"
       },
       "original": {
@@ -17,27 +43,27 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1662099760,
-        "narHash": "sha256-MdZLCTJPeHi/9fg6R9fiunyDwP3XHJqDd51zWWz9px0=",
+        "lastModified": 1683207485,
+        "narHash": "sha256-gs+PHt/y/XQB7S8+YyBLAM8LjgYpPZUVFQBwpFSmJro=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "67e45078141102f45eff1589a831aeaa3182b41e",
+        "rev": "cc45a3f8c98e1c33ca996e3504adefbf660a72d1",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.05",
+        "ref": "nixos-22.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "nixpkgsUnstable": {
+    "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1662096612,
-        "narHash": "sha256-R+Q8l5JuyJryRPdiIaYpO5O3A55rT+/pItBrKcy7LM4=",
+        "lastModified": 1683353485,
+        "narHash": "sha256-Skp5El3egmoXPiINWjnoW0ktVfB7PR/xc4F4bhD+BJY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "21de2b973f9fee595a7a1ac4693efff791245c34",
+        "rev": "caf436a52b25164b71e0d48b671127ac2e2a5b75",
         "type": "github"
       },
       "original": {
@@ -49,9 +75,25 @@
     },
     "root": {
       "inputs": {
+        "extra-container": "extra-container",
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
-        "nixpkgsUnstable": "nixpkgsUnstable"
+        "nixpkgs-unstable": "nixpkgs-unstable"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
       }
     }
   },

flake.nix

@@ -5,27 +5,42 @@
   '';
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05";
-    nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11";
+    nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
+    extra-container = {
+      url = "github:erikarvstedt/extra-container";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
   };
 
-  outputs = { self, nixpkgs, nixpkgsUnstable, flake-utils }:
+  outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, ... }:
     let
       supportedSystems = [
         "x86_64-linux"
-        "i686-linux"
         "aarch64-linux"
-        "armv7l-linux"
+        # On these 32-bit platforms, Python pkg `pymemcache` 4.0.0 (required by
+        # `joinmarket`) is broken:
+        # "i686-linux"
+        # "armv7l-linux"
       ];
+
+      test = import ./test/tests.nix nixpkgs.lib;
     in {
       lib = {
         mkNbPkgs = {
           system
           , pkgs ? nixpkgs.legacyPackages.${system}
-          , pkgsUnstable ? nixpkgsUnstable.legacyPackages.${system}
+          , pkgsUnstable ? nixpkgs-unstable.legacyPackages.${system}
         }:
           import ./pkgs { inherit pkgs pkgsUnstable; };
+
+        test = {
+          inherit (test) scenarios;
+        };
+
+        inherit supportedSystems;
       };
 
       overlays.default = final: prev: let
@@ -91,7 +106,12 @@
         # Allow accessing the whole nested `nbPkgs` attrset (including `modulesPkgs`)
         # via this flake.
         # `packages` is not allowed to contain nested pkgs attrsets.
-        legacyPackages = nbPkgs;
+        legacyPackages =
+          nbPkgs //
+          (test.pkgs self pkgs) //
+          {
+            extra-container = self.inputs.extra-container.packages.${system}.default;
+          };
 
         apps = rec {
           default = vm;
@@ -102,6 +122,8 @@
             program = toString packages.runVM;
           };
         };
+
+        devShells.default = import ./dev/dev-env/dev-shell.nix pkgs;
       }
     ));
 }

helper/start-bash-session.sh

@@ -0,0 +1,12 @@
+# Start an interactive bash session in the current bash environment.
+# This is helpful for debugging bash scripts like pkg update scripts,
+# by adding `source <path-to>/start-bash-session.sh` at the location to
+# be inspected.
+
+
+# BASH_ENVIRONMENT contains definitions of read-only variables like 'BASHOPTS' that
+# cause warnings on evaluation. Suppress these warnings while sourcing.
+#
+# shellcheck disable=SC2016
+BASH_ENVIRONMENT=<(declare -p; declare -pf) \
+  bash --rcfile <(echo 'source $BASH_ENVIRONMENT 2>/dev/null')

helper/update-flake.sh

@@ -12,6 +12,19 @@ set -euo pipefail
 #     pinned to stable.
 #     All other pkgs are pinned to unstable.
 
+forceRun=
+nixosVersion=
+for arg in "$@"; do
+    case $arg in
+        -f)
+            forceRun=1
+            ;;
+        *)
+            nixosVersion=$arg
+            ;;
+    esac
+done
+
 # cd to script dir
 cd "${BASH_SOURCE[0]%/*}"
 
@@ -21,7 +34,7 @@ if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then
     exit 1
 fi
 
-if [[ ${1:-} != -f ]] && ! git diff --quiet ../flake.{nix,lock}; then
+if [[ $forceRun ]] && ! git diff --quiet ../flake.{nix,lock}; then
     echo "error: flake.nix/flake.lock have changes. Run with option -f to ignore."
     exit 1
 fi
@@ -36,6 +49,9 @@ versions=$(nix eval --json -f update-flake.nix versions)
 # versions=$(echo "$versions" | sed 's|1|0|g')
 
 echo "Updating main flake"
+if [[ $nixosVersion ]]; then
+    sed -Ei "s|(nixpkgs.url = .*nixos-)[^\"]+|\1$nixosVersion|" ../flake.nix
+fi
 nix flake update ..
 
 echo

modules/backups.nix

@@ -6,7 +6,7 @@ let
     enable = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Enable backups of node data.
         This uses the NixOS duplicity service.
         To further configure the backup, you can set NixOS options `services.duplicity.*`.
@@ -16,34 +16,34 @@ let
     with-bulk-data = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Whether to also backup Bitcoin blockchain and other bulk data.
       '';
     };
     destination = mkOption {
       type = types.str;
       default = "file:///var/lib/localBackups";
-      description = ''
+      description = mdDoc ''
         Where to back up to.
       '';
     };
     frequency = mkOption {
       type = types.nullOr types.str;
       default = null;
-      description = ''
+      description = mdDoc ''
         Run backup with the given frequency. If null, do not run automatically.
       '';
     };
     postgresqlDatabases = mkOption {
       type = types.listOf types.str;
       default = [];
-      description = "List of database names to backup.";
+      description = mdDoc "List of database names to backup.";
     };
     extraFiles = mkOption {
       type = types.listOf types.str;
       default = [];
       example = [ "/var/lib/nginx" ];
-      description = "Additional files to be appended to filelist.";
+      description = mdDoc "Additional files to be appended to filelist.";
     };
   };
 

modules/bitcoind.nix

@@ -8,19 +8,19 @@ let
       address = mkOption {
         type = types.str;
         default = "127.0.0.1";
-        description = "Address to listen for peer connections.";
+        description = mdDoc "Address to listen for peer connections.";
       };
       port = mkOption {
         type = types.port;
         default = 8333;
-        description = "Port to listen for peer connections.";
+        description = mdDoc "Port to listen for peer connections.";
       };
       onionPort = mkOption {
         type = types.nullOr types.port;
         # When the bitcoind onion service is enabled, add an onion-tagged socket
         # to distinguish local connections from Tor connections
         default = if (config.nix-bitcoin.onionServices.bitcoind.enable or false) then 8334 else null;
-        description = ''
+        description = mdDoc ''
           Port to listen for Tor peer connections.
           If set, inbound connections to this port are tagged as onion peers.
         '';
@@ -28,15 +28,15 @@ let
       listen = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = mdDoc ''
           Listen for peer connections at `address:port`
-          and `address:onionPort` (if `onionPort` is set).
+          and `address:onionPort` (if {option}`onionPort` is set).
         '';
       };
       listenWhitelisted = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = mdDoc ''
           Listen for peer connections at `address:whitelistedPort`.
           Peers connected through this socket are automatically whitelisted.
         '';
@@ -44,12 +44,12 @@ let
       whitelistedPort = mkOption {
         type = types.port;
         default = 8335;
-        description = "See `listenWhitelisted`.";
+        description = mdDoc "See `listenWhitelisted`.";
       };
       getPublicAddressCmd = mkOption {
         type = types.str;
         default = "";
-        description = ''
+        description = mdDoc ''
           Bash expression which outputs the public service address to announce to peers.
           If left empty, no address is announced.
         '';
@@ -58,7 +58,7 @@ let
         type = types.package;
         default = config.nix-bitcoin.pkgs.bitcoind;
         defaultText = "config.nix-bitcoin.pkgs.bitcoind";
-        description = "The package providing bitcoin binaries.";
+        description = mdDoc "The package providing bitcoin binaries.";
       };
       extraConfig = mkOption {
         type = types.lines;
@@ -67,41 +67,41 @@ let
           par=16
           logips=1
         '';
-        description = "Extra lines appended to <filename>bitcoin.conf</filename>.";
+        description = mdDoc "Extra lines appended to {file}`bitcoin.conf`.";
       };
       dataDir = mkOption {
         type = types.path;
         default = "/var/lib/bitcoind";
-        description = "The data directory for bitcoind.";
+        description = mdDoc "The data directory for bitcoind.";
       };
       rpc = {
         address = mkOption {
           type = types.str;
           default = "127.0.0.1";
-          description = ''
+          description = mdDoc ''
             Address to listen for JSON-RPC connections.
           '';
         };
         port = mkOption {
           type = types.port;
           default = 8332;
-          description = "Port to listen for JSON-RPC connections.";
+          description = mdDoc "Port to listen for JSON-RPC connections.";
         };
         threads = mkOption {
           type = types.nullOr types.ints.u16;
           default = null;
-          description = "The number of threads to service RPC calls.";
+          description = mdDoc "The number of threads to service RPC calls.";
         };
         allowip = mkOption {
           type = types.listOf types.str;
           default = [ "127.0.0.1" ];
-          description = ''
+          description = mdDoc ''
             Allow JSON-RPC connections from specified sources.
           '';
         };
         users = mkOption {
           default = {};
-          description = ''
+          description = mdDoc ''
             Allowed users for JSON-RPC connections.
           '';
           example = {
@@ -116,16 +116,16 @@ let
                 type = types.str;
                 default = name;
                 example = "alice";
-                description = ''
+                description = mdDoc ''
                   Username for JSON-RPC connections.
                 '';
               };
               passwordHMAC = mkOption {
                 type = types.str;
                 example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae";
-                description = ''
+                description = mdDoc ''
                   Password HMAC-SHA-256 for JSON-RPC connections. Must be a string of the
-                  format `salt-hex$hmac-hex`.
+                  format `<SALT-HEX>$<HMAC-HEX>`.
                 '';
               };
               passwordHMACFromFile = mkOption {
@@ -136,7 +136,7 @@ let
               rpcwhitelist = mkOption {
                 type = types.listOf types.str;
                 default = [];
-                description = ''
+                description = mdDoc ''
                   List of allowed rpc calls for each user.
                   If empty list, rpcwhitelist is disabled for that user.
                 '';
@@ -148,7 +148,7 @@ let
       regtest = mkOption {
         type = types.bool;
         default = false;
-        description = "Enable regtest mode.";
+        description = mdDoc "Enable regtest mode.";
       };
       network = mkOption {
         readOnly = true;
@@ -161,12 +161,12 @@ let
       proxy = mkOption {
         type = types.nullOr types.str;
         default = if cfg.tor.proxy then config.nix-bitcoin.torClientAddressWithPort else null;
-        description = "Connect through SOCKS5 proxy";
+        description = mdDoc "Connect through SOCKS5 proxy";
       };
       i2p = mkOption {
         type = types.enum [ false true "only-outgoing" ];
         default = false;
-        description = ''
+        description = mdDoc ''
           Enable peer connections via i2p.
           With `only-outgoing`, incoming i2p connections are disabled.
         '';
@@ -174,7 +174,7 @@ let
       dataDirReadableByGroup = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = mdDoc ''
           If enabled, data dir content is readable by the bitcoind service group.
           Warning: This disables bitcoind's wallet support.
         '';
@@ -182,7 +182,7 @@ let
       sysperms = mkOption {
         type = types.nullOr types.bool;
         default = null;
-        description = ''
+        description = mdDoc ''
           Create new files with system default permissions, instead of umask 077
           (only effective with disabled wallet functionality)
         '';
@@ -190,7 +190,7 @@ let
       disablewallet = mkOption {
         type = types.nullOr types.bool;
         default = null;
-        description = ''
+        description = mdDoc ''
           Do not load the wallet and disable wallet RPC calls
         '';
       };
@@ -198,13 +198,13 @@ let
         type = types.nullOr (types.ints.between 4 16384);
         default = null;
         example = 4000;
-        description = "Override the default database cache size in MiB.";
+        description = mdDoc "Override the default database cache size in MiB.";
       };
       prune = mkOption {
         type = types.ints.unsigned;
         default = 0;
         example = 10000;
-        description = ''
+        description = mdDoc ''
           Automatically prune block files to stay under the specified target size in MiB.
           Value 0 disables pruning.
         '';
@@ -212,25 +212,25 @@ let
       txindex = mkOption {
         type = types.bool;
         default = false;
-        description = "Enable the transaction index.";
+        description = mdDoc "Enable the transaction index.";
       };
       zmqpubrawblock = mkOption {
         type = types.nullOr types.str;
         default = null;
         example = "tcp://127.0.0.1:28332";
-        description = "ZMQ address for zmqpubrawblock notifications";
+        description = mdDoc "ZMQ address for zmqpubrawblock notifications";
       };
       zmqpubrawtx = mkOption {
         type = types.nullOr types.str;
         default = null;
         example = "tcp://127.0.0.1:28333";
-        description = "ZMQ address for zmqpubrawtx notifications";
+        description = mdDoc "ZMQ address for zmqpubrawtx notifications";
       };
       assumevalid = mkOption {
         type = types.nullOr types.str;
         default = null;
         example = "00000000000000000000e5abc3a74fe27dc0ead9c70ea1deb456f11c15fd7bc6";
-        description = ''
+        description = mdDoc ''
           If this block is in the chain assume that it and its ancestors are
           valid and potentially skip their script verification.
         '';
@@ -239,37 +239,37 @@ let
         type = types.listOf types.str;
         default = [];
         example = [ "ecoc5q34tmbq54wl.onion" ];
-        description = "Add nodes to connect to and attempt to keep the connections open";
+        description = mdDoc "Add nodes to connect to and attempt to keep the connections open";
       };
       discover = mkOption {
         type = types.nullOr types.bool;
         default = null;
-        description = "Discover own IP addresses";
+        description = mdDoc "Discover own IP addresses";
       };
       addresstype = mkOption {
         type = types.nullOr types.str;
         default = null;
         example = "bech32";
-        description = "The type of addresses to use";
+        description = mdDoc "The type of addresses to use";
       };
       user = mkOption {
         type = types.str;
         default = "bitcoin";
-        description = "The user as which to run bitcoind.";
+        description = mdDoc "The user as which to run bitcoind.";
       };
       group = mkOption {
         type = types.str;
         default = cfg.user;
-        description = "The group as which to run bitcoind.";
+        description = mdDoc "The group as which to run bitcoind.";
       };
       cli = mkOption {
         readOnly = true;
         type = types.package;
-        default = pkgs.writeScriptBin "bitcoin-cli" ''
+        default = pkgs.writers.writeBashBin "bitcoin-cli" ''
           exec ${cfg.package}/bin/bitcoin-cli -datadir='${cfg.dataDir}' "$@"
         '';
         defaultText = "(See source)";
-        description = "Binary to connect with the bitcoind instance.";
+        description = mdDoc "Binary to connect with the bitcoind instance.";
       };
       tor = nbLib.tor;
     };
@@ -367,9 +367,10 @@ in {
       proto.sam.enable = true;
     };
 
-    systemd.tmpfiles.rules = [
-      "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
-    ];
+    # Commented out to avoid trying to chown the nfs-mounted directory
+    # systemd.tmpfiles.rules = [
+    #   "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
+    # ];
 
     systemd.services.bitcoind = {
       # Use `wants` instead of `requires` so that bitcoind and all dependent services
@@ -380,7 +381,7 @@ in {
       # TODO-EXTERNAL: Instead of `wants`, use a future systemd dependency type
       # that propagates initial start failures but no restarts
       wants = [ "nix-bitcoin-secrets.target" ];
-      after = [ "network.target" "nix-bitcoin-secrets.target" ];
+      after = [ "network-online.target" "nix-bitcoin-secrets.target" ];
       wantedBy = [ "multi-user.target" ];
 
       preStart = let
@@ -412,21 +413,22 @@ in {
       '';
 
       # Enable RPC access for group
-      postStart = ''
-        chmod g=r '${cfg.dataDir}/${optionalString cfg.regtest "regtest/"}.cookie'
-      '';
+      # postStart = ''
+      #   chmod g=r '${cfg.dataDir}/${optionalString cfg.regtest "regtest/"}.cookie'
+      # '';
 
       serviceConfig = nbLib.defaultHardening // {
         Type = "notify";
         NotifyAccess = "all";
         User = cfg.user;
         Group = cfg.group;
-        TimeoutStartSec = "10min";
-        TimeoutStopSec = "10min";
+        TimeoutStartSec = "30min";
+        TimeoutStopSec = "30min";
         ExecStart = "${cfg.package}/bin/bitcoind -datadir='${cfg.dataDir}'";
         Restart = "on-failure";
         UMask = mkIf cfg.dataDirReadableByGroup "0027";
-        ReadWritePaths = [ cfg.dataDir ];
+        #ReadWritePaths = [ cfg.dataDir ];
+        ReadWritePaths = [ "/dummy" ];
       } // nbLib.allowedIPAddresses cfg.tor.enforce
         // optionalAttrs zmqServerEnabled nbLib.allowNetlink;
     };

modules/btcpayserver.nix

@@ -8,12 +8,12 @@ let
       address = mkOption {
         type = types.str;
         default = "127.0.0.1";
-        description = "Address to listen on.";
+        description = mdDoc "Address to listen on.";
       };
       port = mkOption {
         type = types.port;
         default = 23000;
-        description = "Port to listen on.";
+        description = mdDoc "Port to listen on.";
       };
       package = mkOption {
         type = types.package;
@@ -22,38 +22,38 @@ let
                   else
                     config.nix-bitcoin.pkgs.btcpayserver;
         defaultText = "(See source)";
-        description = "The package providing btcpayserver binaries.";
+        description = mdDoc "The package providing btcpayserver binaries.";
       };
       dataDir = mkOption {
         type = types.path;
         default = "/var/lib/btcpayserver";
-        description = "The data directory for btcpayserver.";
+        description = mdDoc "The data directory for btcpayserver.";
       };
       lightningBackend = mkOption {
         type = types.nullOr (types.enum [ "clightning" "lnd" ]);
         default = null;
-        description = "The lightning node implementation to use.";
+        description = mdDoc "The lightning node implementation to use.";
       };
       lbtc = mkOption {
         type = types.bool;
         default = false;
-        description = "Enable liquid support in btcpayserver.";
+        description = mdDoc "Enable liquid support in btcpayserver.";
       };
       rootpath = mkOption {
         type = types.nullOr types.str;
         default = null;
         example = "btcpayserver";
-        description = "The prefix for root-relative btcpayserver URLs.";
+        description = mdDoc "The prefix for root-relative btcpayserver URLs.";
       };
       user = mkOption {
         type = types.str;
         default = "btcpayserver";
-        description = "The user as which to run btcpayserver.";
+        description = mdDoc "The user as which to run btcpayserver.";
       };
       group = mkOption {
         type = types.str;
         default = cfg.btcpayserver.user;
-        description = "The group as which to run btcpayserver.";
+        description = mdDoc "The group as which to run btcpayserver.";
       };
       tor.enforce = nbLib.tor.enforce;
     };
@@ -63,7 +63,7 @@ let
         # This option is only used by netns-isolation
         internal = true;
         default = cfg.btcpayserver.enable;
-        description = ''
+        description = mdDoc ''
           nbxplorer is always enabled when btcpayserver is enabled.
         '';
       };
@@ -71,32 +71,32 @@ let
         type = types.package;
         default = config.nix-bitcoin.pkgs.nbxplorer;
         defaultText = "config.nix-bitcoin.pkgs.nbxplorer";
-        description = "The package providing nbxplorer binaries.";
+        description = mdDoc "The package providing nbxplorer binaries.";
       };
       address = mkOption {
         type = types.str;
         default = "127.0.0.1";
-        description = "Address to listen on.";
+        description = mdDoc "Address to listen on.";
       };
       port = mkOption {
         type = types.port;
         default = 24444;
-        description = "Port to listen on.";
+        description = mdDoc "Port to listen on.";
       };
       dataDir = mkOption {
         type = types.path;
         default = "/var/lib/nbxplorer";
-        description = "The data directory for nbxplorer.";
+        description = mdDoc "The data directory for nbxplorer.";
       };
       user = mkOption {
         type = types.str;
         default = "nbxplorer";
-        description = "The user as which to run nbxplorer.";
+        description = mdDoc "The user as which to run nbxplorer.";
       };
       group = mkOption {
         type = types.str;
         default = cfg.nbxplorer.user;
-        description = "The group as which to run nbxplorer.";
+        description = mdDoc "The group as which to run nbxplorer.";
       };
       tor.enforce = nbLib.tor.enforce;
     };
@@ -193,14 +193,14 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = [ cfg.nbxplorer.dataDir ];
-        MemoryDenyWriteExecute = "false";
+        MemoryDenyWriteExecute = false;
       } // nbLib.allowedIPAddresses cfg.nbxplorer.tor.enforce;
     };
 
     systemd.services.btcpayserver = let
       nbExplorerUrl = "http://${nbLib.addressWithPort cfg.nbxplorer.address cfg.nbxplorer.port}/";
       nbExplorerCookie = "${cfg.nbxplorer.dataDir}/${bitcoind.makeNetworkName "Main" "RegTest"}/.cookie";
-      configFile = builtins.toFile "config" (''
+      configFile = builtins.toFile "btcpayserver-config" (''
         network=${bitcoind.network}
         bind=${cfg.btcpayserver.address}
         port=${toString cfg.btcpayserver.port}
@@ -212,42 +212,40 @@ in {
         rootpath=${cfg.btcpayserver.rootpath}
       '' + optionalString (cfg.btcpayserver.lightningBackend == "clightning") ''
         btclightning=type=clightning;server=unix:///${cfg.clightning.dataDir}/${bitcoind.makeNetworkName "bitcoin" "regtest"}/lightning-rpc
-      '' + optionalString cfg.btcpayserver.lbtc ''
+      '' + optionalString (cfg.btcpayserver.lightningBackend == "lnd")
+        (
+          "btclightning=type=lnd-rest;" +
+          "server=https://${cfg.lnd.restAddress}:${toString cfg.lnd.restPort}/;" +
+          "macaroonfilepath=/run/lnd/btcpayserver.macaroon;" +
+          "certfilepath=${config.services.lnd.certPath}" +
+          "\n"
+        )
+      + optionalString cfg.btcpayserver.lbtc ''
         chains=btc,lbtc
         lbtcexplorerurl=${nbExplorerUrl}
         lbtcexplorercookiefile=${nbExplorerCookie}
       '');
-      lndConfig =
-        "btclightning=type=lnd-rest;" +
-        "server=https://${cfg.lnd.restAddress}:${toString cfg.lnd.restPort}/;" +
-        "macaroonfilepath=/run/lnd/btcpayserver.macaroon;" +
-        "certthumbprint=";
     in let self = {
       wantedBy = [ "multi-user.target" ];
       requires = [ "nbxplorer.service" "postgresql.service" ]
                  ++ optional (cfg.btcpayserver.lightningBackend != null) "${cfg.btcpayserver.lightningBackend}.service";
       after = self.requires;
-      preStart = ''
-        install -m 600 ${configFile} '${cfg.btcpayserver.dataDir}/settings.config'
-        ${optionalString (cfg.btcpayserver.lightningBackend == "lnd") ''
-          {
-            echo -n "${lndConfig}"
-            ${pkgs.openssl}/bin/openssl x509 -noout -fingerprint -sha256 -in ${config.services.lnd.certPath} \
-              | sed -e 's/.*=//;s/://g'
-          } >> '${cfg.btcpayserver.dataDir}/settings.config'
-        ''}
-      '';
       serviceConfig = nbLib.defaultHardening // {
         ExecStart = ''
-          ${cfg.btcpayserver.package}/bin/btcpayserver --conf='${cfg.btcpayserver.dataDir}/settings.config' \
+          ${cfg.btcpayserver.package}/bin/btcpayserver --conf=${configFile} \
             --datadir='${cfg.btcpayserver.dataDir}'
         '';
         User = cfg.btcpayserver.user;
-        Restart = "on-failure";
-        RestartSec = "10s";
+        # Also restart after the program has exited successfully.
+        # This is required to support restarting from the web interface after
+        # interactive plugin installation.
+        # Restart rate limiting is implemented via the `startLimit*` options below.
+        Restart = "always";
         ReadWritePaths = [ cfg.btcpayserver.dataDir ];
-        MemoryDenyWriteExecute = "false";
+        MemoryDenyWriteExecute = false;
       } // nbLib.allowedIPAddresses cfg.btcpayserver.tor.enforce;
+      startLimitIntervalSec = 30;
+      startLimitBurst = 10;
     }; in self;
 
     users.users.${cfg.nbxplorer.user} = {

modules/charge-lnd.nix

@@ -9,18 +9,17 @@ let
       type = listOf str;
       default = [];
       example = [ "--verbose" "--dry-run" ];
-      description = "Extra flags to pass to the charge-lnd command.";
+      description = mdDoc "Extra flags to pass to the charge-lnd command.";
     };
 
     interval = mkOption {
       type = str;
       default = "*-*-* 04:00:00";
       example = "hourly";
-      description = ''
+      description = mdDoc ''
         Systemd calendar expression when to adjust fees.
 
-        See <citerefentry><refentrytitle>systemd.time</refentrytitle>
-        <manvolnum>7</manvolnum></citerefentry> for possible values.
+        See {man}`systemd.time(7)` for possible values.
 
         Default is once a day.
       '';
@@ -29,7 +28,7 @@ let
     randomDelay = mkOption {
       type = str;
       default = "1h";
-      description = ''
+      description = mdDoc ''
         Random delay to add to scheduled time.
       '';
     };
@@ -55,7 +54,7 @@ let
         [default]
         strategy = ignore
       '';
-      description = ''
+      description = mdDoc ''
         Policy definitions in INI format.
 
         See https://github.com/accumulator/charge-lnd/blob/master/README.md#usage
@@ -126,7 +125,6 @@ in
     };
 
     systemd.timers.charge-lnd = {
-      description = "Adjust LND routing fees";
       wantedBy = [ "timers.target" ];
       timerConfig = {
         OnCalendar = cfg.interval;

modules/clightning-plugins/clboss.nix

@@ -7,7 +7,7 @@ let cfg = config.services.clightning.plugins.clboss; in
     enable = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Whether to enable CLBOSS (clightning plugin).
         See also: https://github.com/ZmnSCPxj/clboss#operating
       '';
@@ -15,36 +15,36 @@ let cfg = config.services.clightning.plugins.clboss; in
     min-onchain = mkOption {
       type = types.ints.positive;
       default = 30000;
-      description = ''
+      description = mdDoc ''
         Target amount (in satoshi) that CLBOSS will leave on-chain.
-        clboss will only open new channels if this amount is smaller than
-        the funds in your clightning wallet.
+        clboss will only open new channels if the funds in your clightning wallet are
+        larger than this amount.
       '';
     };
     min-channel = mkOption {
       type = types.ints.positive;
       default = 500000;
-      description = "The minimum size (in satoshi) of channels created by CLBOSS.";
+      description = mdDoc "The minimum size (in satoshi) of channels created by CLBOSS.";
     };
     max-channel = mkOption {
       type = types.ints.positive;
       default = 16777215;
-      description = "The maximum size (in satoshi) of channels created by CLBOSS.";
+      description = mdDoc "The maximum size (in satoshi) of channels created by CLBOSS.";
     };
     zerobasefee = mkOption {
       type = types.enum [ "require" "allow" "disallow" ];
       default = "allow";
-      description = ''
-        require: set `base_fee` to 0.
-        allow: set `base_fee` according to the CLBOSS heuristics, which may include value 0.
-        disallow: set `base_fee` to according to the CLBOSS heuristics, with a minimum value of 1.
+      description = mdDoc ''
+        `require`: set `base_fee` to 0.
+        `allow`: set `base_fee` according to the CLBOSS heuristics, which may include value 0.
+        `disallow`: set `base_fee` to according to the CLBOSS heuristics, with a minimum value of 1.
       '';
     };
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.clboss;
       defaultText = "config.nix-bitcoin.pkgs.clboss";
-      description = "The package providing clboss binaries.";
+      description = mdDoc "The package providing clboss binaries.";
     };
   };
 

modules/clightning-plugins/default.nix

@@ -17,6 +17,7 @@ in {
     ./feeadjuster.nix
     ./prometheus.nix
     ./summary.nix
+    ./trustedcoin.nix
     ./zmq.nix
   ];
 

modules/clightning-plugins/feeadjuster.nix

@@ -6,7 +6,7 @@ let
     enable = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Enable feeaduster (clightning plugin).
         This plugin auto-updates channel fees to keep channels balanced.
 
@@ -18,17 +18,17 @@ let
     fuzz = mkOption {
       type = types.bool;
       default = true;
-      description = "Enable update threshold randomization and hysteresis.";
+      description = mdDoc "Enable update threshold randomization and hysteresis.";
     };
     adjustOnForward = mkOption {
       type = types.bool;
       default = false;
-      description = "Automatically update fees on forward events.";
+      description = mdDoc "Automatically update fees on forward events.";
     };
     method = mkOption {
       type = types.enum [ "soft" "default" "hard" ];
       default = "default";
-      description = ''
+      description = mdDoc ''
         Adjustment method to calculate channel fees.
         `soft`: less difference when adjusting fees.
         `hard`: greater difference when adjusting fees.
@@ -37,7 +37,7 @@ let
     adjustDaily = mkOption {
       type = types.bool;
       default = true;
-      description = "Automatically update fees daily.";
+      description = mdDoc "Automatically update fees daily.";
     };
   };
 

modules/clightning-plugins/summary.nix

@@ -8,22 +8,22 @@ let cfg = config.services.clightning.plugins.summary; in
     currency = mkOption {
       type = types.str;
       default = "USD";
-      description = "The currency to look up on btcaverage.";
+      description = mdDoc "The currency to look up on btcaverage.";
     };
     currencyPrefix = mkOption {
       type = types.str;
       default = "USD $";
-      description = "The prefix to use for the currency.";
+      description = mdDoc "The prefix to use for the currency.";
     };
     availabilityInterval = mkOption {
       type = types.int;
       default = 300;
-      description = "How often in seconds the availability should be calculated.";
+      description = mdDoc "How often in seconds the availability should be calculated.";
     };
     availabilityWindow = mkOption {
       type = types.int;
       default = 72;
-      description = "How many hours the availability should be averaged over.";
+      description = mdDoc "How many hours the availability should be averaged over.";
     };
   };
 

modules/clightning-plugins/trustedcoin.nix

@@ -0,0 +1,28 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let cfg = config.services.clightning.plugins.trustedcoin; in
+{
+  options.services.clightning.plugins.trustedcoin = {
+    enable = mkEnableOption "Trustedcoin (clightning plugin)";
+    package = mkOption {
+      type = types.package;
+      default = config.nix-bitcoin.pkgs.trustedcoin;
+      defaultText = "config.nix-bitcoin.pkgs.trustedcoin";
+      description = mdDoc "The package providing trustedcoin binaries.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${cfg.package}/bin/trustedcoin
+      disable-plugin=bcli
+    '';
+
+    # Trustedcoin does not honor the clightning's proxy configuration.
+    # Ref.: https://github.com/nbd-wtf/trustedcoin/pull/19
+    systemd.services.clightning.environment = mkIf (config.services.clightning.proxy != null) {
+      HTTPS_PROXY = "socks5://${config.services.clightning.proxy}";
+    };
+  };
+}

modules/clightning-plugins/zmq.nix

@@ -25,7 +25,7 @@ let
     mkOption {
       type = types.nullOr types.str;
       default = null;
-      description = "Endpoint for ${name}";
+      description = mdDoc "Endpoint for ${name}";
     };
 
   setEndpoint = ep:

modules/clightning-replication.nix

@@ -6,7 +6,7 @@ let
     enable =  mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Enable live replication of the clightning database.
         This prevents losing off-chain funds when the primary wallet file becomes
         inaccessible.
@@ -26,7 +26,7 @@ let
         type = types.nullOr types.str;
         default = null;
         example = "user@10.0.0.1:directory";
-        description = ''
+        description = mdDoc ''
           The SSH destination for which a SSHFS will be mounted.
           `directory` is relative to the home of `user`.
 
@@ -40,12 +40,12 @@ let
       port = mkOption {
         type = types.port;
         default = 22;
-        description = "SSH port of the remote server.";
+        description = mdDoc "SSH port of the remote server.";
       };
       sshOptions = mkOption {
         type = with types; listOf str;
         default = [ "reconnect" "ServerAliveInterval=50" ];
-        description = "SSH options used for mounting the SSHFS.";
+        description = mdDoc "SSH options used for mounting the SSHFS.";
       };
     };
     local = {
@@ -53,7 +53,7 @@ let
         type = types.nullOr types.path;
         default = null;
         example = "/var/backup/clightning";
-        description = ''
+        description = mdDoc ''
           This option can be specified instead of `sshfs.destination` to enable
           replication to a local directory.
 
@@ -69,7 +69,7 @@ let
       setupDirectory = mkOption {
         type = types.bool;
         default = true;
-        description = ''
+        description = mdDoc ''
           Create `local.directory` if it doesn't exist and set write permissions
           for the `clightning` user.
         '';
@@ -78,10 +78,10 @@ let
     encrypt = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Whether to encrypt the replicated database with gocryptfs.
         The encryption password is automatically generated and stored
-        in file `$secretsDir/clightning-replication-password`.
+        in file {file}`$secretsDir/clightning-replication-password`.
       '';
     };
   };

modules/clightning-rest.nix

@@ -7,17 +7,17 @@ let
     port = mkOption {
       type = types.port;
       default = 3001;
-      description = "REST server port.";
+      description = mdDoc "REST server port.";
     };
     docPort = mkOption {
       type = types.port;
       default = 4001;
-      description = "Swagger API documentation server port.";
+      description = mdDoc "Swagger API documentation server port.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/clightning-rest";
-      description = "The data directory for clightning-rest.";
+      description = mdDoc "The data directory for clightning-rest.";
     };
     extraConfig = mkOption {
       type = types.attrs;
@@ -25,7 +25,7 @@ let
       example = {
         DOMAIN = "mynode.org";
       };
-      description = ''
+      description = mdDoc ''
         Extra config options.
         See: https://github.com/Ride-The-Lightning/c-lightning-REST#option-1-via-config-file-cl-rest-configjson
       '';
@@ -34,7 +34,7 @@ let
     group = mkOption {
       readOnly = true;
       default = clightning.group;
-      description = "The group under which clightning-rest is run.";
+      description = mdDoc "The group under which clightning-rest is run.";
     };
     # Rest server address.
     # Not configurable. The server always listens on all interfaces:

modules/clightning.nix

@@ -7,24 +7,24 @@ let
     address = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = "Address to listen for peer connections.";
+      description = mdDoc "Address to listen for peer connections.";
     };
     port = mkOption {
       type = types.port;
       default = 9735;
-      description = "Port to listen for peer connections.";
+      description = mdDoc "Port to listen for peer connections.";
     };
     proxy = mkOption {
       type = types.nullOr types.str;
       default = if cfg.tor.proxy then config.nix-bitcoin.torClientAddressWithPort else null;
-      description = ''
+      description = mdDoc ''
         Socks proxy for connecting to Tor nodes (or for all connections if option always-use-proxy is set).
       '';
     };
     always-use-proxy = mkOption {
       type = types.bool;
       default = cfg.tor.proxy;
-      description = ''
+      description = mdDoc ''
         Always use the proxy, even to connect to normal IP addresses.
         You can still connect to Unix domain sockets manually.
         This also disables all DNS lookups, to avoid leaking address information.
@@ -33,18 +33,18 @@ let
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/clightning";
-      description = "The data directory for clightning.";
+      description = mdDoc "The data directory for clightning.";
     };
     networkDir = mkOption {
       readOnly = true;
       default = "${cfg.dataDir}/${network}";
-      description = "The network data directory.";
+      description = mdDoc "The network data directory.";
     };
     wallet = mkOption {
       type = types.nullOr types.str;
       default = null;
       example = "sqlite3:///var/lib/clightning/bitcoin/lightningd.sqlite3";
-      description = ''
+      description = mdDoc ''
         Wallet data scheme (sqlite3 or postgres) and location/connection
         parameters, as fully qualified data source name.
       '';
@@ -55,42 +55,42 @@ let
       example = ''
         alias=mynode
       '';
-      description = ''
+      description = mdDoc ''
         Extra lines appended to the configuration file.
 
         See all available options at
         https://github.com/ElementsProject/lightning/blob/master/doc/lightningd-config.5.md
-        or by running `lightningd --help`.
+        or by running {command}`lightningd --help`.
       '';
     };
     user = mkOption {
       type = types.str;
       default = "clightning";
-      description = "The user as which to run clightning.";
+      description = mdDoc "The user as which to run clightning.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run clightning.";
+      description = mdDoc "The group as which to run clightning.";
     };
     package = mkOption {
       type = types.package;
       default = nbPkgs.clightning;
       defaultText = "config.nix-bitcoin.pkgs.clightning";
-      description = "The package providing clightning binaries.";
+      description = mdDoc "The package providing clightning binaries.";
     };
     cli = mkOption {
       readOnly = true;
-      default = pkgs.writeScriptBin "lightning-cli" ''
+      default = pkgs.writers.writeBashBin "lightning-cli" ''
         ${cfg.package}/bin/lightning-cli --lightning-dir='${cfg.dataDir}' "$@"
       '';
       defaultText = "(See source)";
-      description = "Binary to connect with the clightning instance.";
+      description = mdDoc "Binary to connect with the clightning instance.";
     };
     getPublicAddressCmd = mkOption {
       type = types.str;
       default = "";
-      description = ''
+      description = mdDoc ''
         Bash expression which outputs the public service address to announce to peers.
         If left empty, no address is announced.
       '';
@@ -102,19 +102,28 @@ let
   nbLib = config.nix-bitcoin.lib;
   nbPkgs = config.nix-bitcoin.pkgs;
 
-  network = config.services.bitcoind.makeNetworkName "bitcoin" "regtest";
+  inherit (config.services) bitcoind;
+
+  network = bitcoind.makeNetworkName "bitcoin" "regtest";
   configFile = pkgs.writeText "config" ''
     network=${network}
-    bitcoin-datadir=${config.services.bitcoind.dataDir}
+    ${optionalString (!cfg.plugins.trustedcoin.enable) "bitcoin-datadir=${bitcoind.dataDir}"}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     always-use-proxy=${boolToString cfg.always-use-proxy}
     bind-addr=${cfg.address}:${toString cfg.port}
-    bitcoin-rpcconnect=${nbLib.address config.services.bitcoind.rpc.address}
-    bitcoin-rpcport=${toString config.services.bitcoind.rpc.port}
-    bitcoin-rpcuser=${config.services.bitcoind.rpc.users.public.name}
+
+    bitcoin-rpcconnect=${nbLib.address bitcoind.rpc.address}
+    bitcoin-rpcport=${toString bitcoind.rpc.port}
+    bitcoin-rpcuser=${bitcoind.rpc.users.public.name}
+
     rpc-file-mode=0660
     log-timestamps=false
     ${optionalString (cfg.wallet != null) "wallet=${cfg.wallet}"}
+    ${ # TODO-EXTERNAL: When updating from a version of clightning before 22.11
+       # to version 22.11.1, then the database upgrade needs to be allowed
+       # explicitly. Remove this when it's unlikely that this module is used
+       # with a clightning version 22.11.1 package.
+      optionalString (cfg.package.version == "22.11.1") "database-upgrade=true"}
     ${cfg.extraConfig}
   '';
 
@@ -154,6 +163,7 @@ in {
         {
           cat ${configFile}
           echo "bitcoin-rpcpassword=$(cat ${config.nix-bitcoin.secretsDir}/bitcoin-rpcpassword-public)"
+
           ${optionalString (cfg.getPublicAddressCmd != "") ''
             echo "announce-addr=$(${cfg.getPublicAddressCmd}):${toString publicPort}"
           ''}

modules/electrs.nix

@@ -7,37 +7,37 @@ let
     address = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = "Address to listen for RPC connections.";
+      description = mdDoc "Address to listen for RPC connections.";
     };
     port = mkOption {
       type = types.port;
       default = 50001;
-      description = "Port to listen for RPC connections.";
+      description = mdDoc "Port to listen for RPC connections.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/electrs";
-      description = "The data directory for electrs.";
+      description = mdDoc "The data directory for electrs.";
     };
     monitoringPort = mkOption {
       type = types.port;
       default = 4224;
-      description = "Prometheus monitoring port.";
+      description = mdDoc "Prometheus monitoring port.";
     };
     extraArgs = mkOption {
       type = types.separatedString " ";
       default = "";
-      description = "Extra command line arguments passed to electrs.";
+      description = mdDoc "Extra command line arguments passed to electrs.";
     };
     user = mkOption {
       type = types.str;
       default = "electrs";
-      description = "The user as which to run electrs.";
+      description = mdDoc "The user as which to run electrs.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run electrs.";
+      description = mdDoc "The group as which to run electrs.";
     };
     tor.enforce = nbLib.tor.enforce;
   };

modules/fulcrum.nix

@@ -6,7 +6,7 @@ let
     enable = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Enable fulcrum, an Electrum server implemented in C++.
 
         Compared to electrs, fulcrum has a 3x larger database size but
@@ -17,23 +17,23 @@ let
 
         This module disables peering (a distributed list of electrum servers that can
         be queried by clients), but you can manually enable it via option
-        `extraConfig`.
+        {option}`extraConfig`.
       '';
     };
     address = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = "Address to listen for RPC connections.";
+      description = mdDoc "Address to listen for RPC connections.";
     };
     port = mkOption {
       type = types.port;
       default = 50001;
-      description = "Port to listen for RPC connections.";
+      description = mdDoc "Port to listen for RPC connections.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/fulcrum";
-      description = "The data directory for fulcrum.";
+      description = mdDoc "The data directory for fulcrum.";
     };
     extraConfig = mkOption {
       type = types.lines;
@@ -41,7 +41,7 @@ let
       example = ''
         peering = true
       '';
-      description = ''
+      description = mdDoc ''
         Extra lines appended to the configuration file.
 
         See all available options at
@@ -51,12 +51,12 @@ let
     user = mkOption {
       type = types.str;
       default = "fulcrum";
-      description = "The user as which to run fulcrum.";
+      description = mdDoc "The user as which to run fulcrum.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run fulcrum.";
+      description = mdDoc "The group as which to run fulcrum.";
     };
     tor.enforce = nbLib.tor.enforce;
   };
@@ -126,6 +126,7 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
+        ProcSubset = "all"; # Fulcrum requires read access to /proc/meminfo
       } // nbLib.allowedIPAddresses cfg.tor.enforce;
     };
 

modules/hardware-wallets.nix

@@ -6,21 +6,21 @@ let
     ledger = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         If enabled, the ledger udev rules will be installed.
       '';
     };
     trezor = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         If enabled, the trezor udev rules will be installed.
       '';
     };
     group = mkOption {
       type = types.str;
       default = "hardware-wallets";
-      description = ''
+      description = mdDoc ''
         Group the hardware wallet udev rules apply to.
       '';
     };

modules/joinmarket-ob-watcher.nix

@@ -7,27 +7,27 @@ let
     address = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = "HTTP server address.";
+      description = mdDoc "HTTP server address.";
     };
     port = mkOption {
       type = types.port;
       default = 62601;
-      description = "HTTP server port.";
+      description = mdDoc "HTTP server port.";
     };
     dataDir = mkOption {
       readOnly = true;
       default = "/var/lib/joinmarket-ob-watcher";
-      description = "The data directory for JoinMarket orderbook watcher.";
+      description = mdDoc "The data directory for JoinMarket orderbook watcher.";
     };
     user = mkOption {
       type = types.str;
       default = "joinmarket-ob-watcher";
-      description = "The user as which to run JoinMarket.";
+      description = mdDoc "The user as which to run JoinMarket.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run JoinMarket.";
+      description = mdDoc "The group as which to run JoinMarket.";
     };
     # This option is only used by netns-isolation.
     # Tor is always enabled.

modules/joinmarket.nix

@@ -7,7 +7,7 @@ let
     payjoinAddress = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = ''
+      description = mdDoc ''
         The address where payjoin onion connections are forwarded to.
         This address is never used directly, it only serves as the internal endpoint
         for the payjoin onion service.
@@ -18,12 +18,12 @@ let
     payjoinPort = mkOption {
       type = types.port;
       default = 64180; # A random private port
-      description = "The port corresponding to option `payjoinAddress`.";
+      description = mdDoc "The port corresponding to option {option}`payjoinAddress`.";
     };
     messagingAddress = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = ''
+      description = mdDoc ''
         The address where messaging onion connections are forwarded to.
         This address is never used directly, it only serves as the internal endpoint
         for the messaging onion service.
@@ -33,29 +33,29 @@ let
     messagingPort = mkOption {
       type = types.port;
       default = 64181; # payjoinPort + 1
-      description = "The port corresponding to option `messagingAddress`.";
+      description = mdDoc "The port corresponding to option {option}`messagingAddress`.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/joinmarket";
-      description = "The data directory for JoinMarket.";
+      description = mdDoc "The data directory for JoinMarket.";
     };
     rpcWalletFile = mkOption {
       type = types.nullOr types.str;
       default = "jm_wallet";
-      description = ''
+      description = mdDoc ''
         Name of the watch-only bitcoind wallet the JoinMarket addresses are imported to.
       '';
     };
     user = mkOption {
       type = types.str;
       default = "joinmarket";
-      description = "The user as which to run JoinMarket.";
+      description = mdDoc "The user as which to run JoinMarket.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run JoinMarket.";
+      description = mdDoc "The group as which to run JoinMarket.";
     };
     cli = mkOption {
       default = cli;
@@ -77,57 +77,57 @@ let
       ordertype = mkOption {
         type = types.enum [ "reloffer" "absoffer" ];
         default = "reloffer";
-        description = ''
-          Which fee type to actually use
+        description = mdDoc ''
+          Which fee type to actually use.
         '';
       };
       cjfee_a = mkOption {
         type = types.ints.unsigned;
         default = 500;
-        description = ''
-          Absolute offer fee you wish to receive for coinjoins (cj) in Satoshis
+        description = mdDoc ''
+          Absolute offer fee you wish to receive for coinjoins (cj) in Satoshis.
         '';
       };
       cjfee_r = mkOption {
         type = types.float;
         default = 0.00002;
-        description = ''
-          Relative offer fee you wish to receive based on a cj's amount
+        description = mdDoc ''
+          Relative offer fee you wish to receive based on a cj's amount.
         '';
       };
       cjfee_factor = mkOption {
         type = types.float;
         default = 0.1;
-        description = ''
-          Variance around the average cj fee
+        description = mdDoc ''
+          Variance around the average cj fee.
         '';
       };
       txfee = mkOption {
         type = types.ints.unsigned;
         default = 100;
-        description = ''
-          The average transaction fee you're adding to coinjoin transactions
+        description = mdDoc ''
+          The average transaction fee you're adding to coinjoin transactions.
         '';
       };
       txfee_contribution_factor = mkOption {
         type = types.float;
         default = 0.3;
-        description = ''
-          Variance around the average tx fee
+        description = mdDoc ''
+          Variance around the average tx fee.
         '';
       };
       minsize = mkOption {
         type = types.ints.unsigned;
         default = 100000;
-        description = ''
+        description = mdDoc ''
           Minimum size of your cj offer in Satoshis. Lower cj amounts will be disregarded.
         '';
       };
       size_factor = mkOption {
         type = types.float;
         default = 0.1;
-        description = ''
-          Variance around all offer sizes
+        description = mdDoc ''
+          Variance around all offer sizes.
         '';
       };
     };
@@ -158,7 +158,7 @@ let
     onion_serving_host = ${cfg.messagingAddress}
     onion_serving_port = ${toString cfg.messagingPort}
     hidden_service_dir =
-    directory_nodes = 3kxw6lf5vf6y26emzwgibzhrzhmhqiw6ekrek3nqfjjmhwznb2moonad.onion:5222,jmdirjmioywe2s5jad7ts6kgcqg66rj6wujj6q77n6wbdrgocqwexzid.onion:5222,bqlpq6ak24mwvuixixitift4yu42nxchlilrcqwk2ugn45tdclg42qid.onion:5222
+    directory_nodes = g3hv4uynnmynqqq2mchf3fcm3yd46kfzmcdogejuckgwknwyq5ya6iad.onion:5222,3kxw6lf5vf6y26emzwgibzhrzhmhqiw6ekrek3nqfjjmhwznb2moonad.onion:5222,bqlpq6ak24mwvuixixitift4yu42nxchlilrcqwk2ugn45tdclg42qid.onion:5222
 
     # irc.darkscience.net
     [MESSAGING:server1]

modules/lightning-loop.nix

@@ -7,56 +7,56 @@ let
     rpcAddress = mkOption {
        type = types.str;
        default = "localhost";
-       description = "Address to listen for gRPC connections.";
+       description = mdDoc "Address to listen for gRPC connections.";
     };
     rpcPort = mkOption {
        type = types.port;
        default = 11010;
-       description = "Port to listen for gRPC connections.";
+       description = mdDoc "Port to listen for gRPC connections.";
     };
     restAddress = mkOption {
        type = types.str;
        default = cfg.rpcAddress;
-       description = "Address to listen for REST connections.";
+       description = mdDoc "Address to listen for REST connections.";
     };
     restPort = mkOption {
        type = types.port;
        default = 8081;
-       description = "Port to listen for REST connections.";
+       description = mdDoc "Port to listen for REST connections.";
     };
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.lightning-loop;
       defaultText = "config.nix-bitcoin.pkgs.lightning-loop";
-      description = "The package providing lightning-loop binaries.";
+      description = mdDoc "The package providing lightning-loop binaries.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/lightning-loop";
-      description = "The data directory for lightning-loop.";
+      description = mdDoc "The data directory for lightning-loop.";
     };
     proxy = mkOption {
       type = types.nullOr types.str;
       default = if cfg.tor.proxy then config.nix-bitcoin.torClientAddressWithPort else null;
-      description = "host:port of SOCKS5 proxy for connnecting to the loop server.";
+      description = mdDoc "`host:port` of SOCKS5 proxy for connnecting to the loop server.";
     };
     certificate = {
       extraIPs = mkOption {
         type = with types; listOf str;
         default = [];
         example = [ "60.100.0.1" ];
-        description = ''
+        description = mdDoc ''
           Extra `subjectAltName` IPs added to the certificate.
-          This works the same as loop option `tlsextraip`.
+          This works the same as loop option {option}`tlsextraip`.
         '';
       };
       extraDomains = mkOption {
         type = with types; listOf str;
         default = [];
         example = [ "example.com" ];
-        description = ''
+        description = mdDoc ''
           Extra `subjectAltName` domain names added to the certificate.
-          This works the same as loop option `tlsextradomain`.
+          This works the same as loop option {option}`tlsextradomain`.
         '';
       };
     };
@@ -66,21 +66,21 @@ let
       example = ''
         debuglevel=trace
       '';
-      description = ''
+      description = mdDoc ''
         Extra lines appended to the configuration file.
         See here for all available options:
         https://github.com/lightninglabs/loop/blob/11ab596080e9d36f1df43edbeba0702b25aa7457/loopd/config.go#L119
       '';
     };
     cli = mkOption {
-      default = pkgs.writeScriptBin "loop" ''
+      default = pkgs.writers.writeBashBin "loop" ''
         ${cfg.package}/bin/loop \
         --rpcserver ${nbLib.addressWithPort cfg.rpcAddress cfg.rpcPort} \
         --macaroonpath '${cfg.dataDir}/${network}/loop.macaroon' \
         --tlscertpath '${secretsDir}/loop-cert' "$@"
       '';
       defaultText = "(See source)";
-      description = "Binary to connect with the lightning-loop instance.";
+      description = mdDoc "Binary to connect with the lightning-loop instance.";
     };
     tor = nbLib.tor;
   };

modules/lightning-pool.nix

@@ -7,38 +7,38 @@ let
     rpcAddress = mkOption {
        type = types.str;
        default = "localhost";
-       description = "Address to listen for gRPC connections.";
+       description = mdDoc "Address to listen for gRPC connections.";
     };
     rpcPort = mkOption {
        type = types.port;
        default = 12010;
-       description = "Port to listen for gRPC connections.";
+       description = mdDoc "Port to listen for gRPC connections.";
     };
     restAddress = mkOption {
        type = types.str;
        default = cfg.rpcAddress;
-       description = "Address to listen for REST connections.";
+       description = mdDoc "Address to listen for REST connections.";
     };
     restPort = mkOption {
        type = types.port;
        default = 8281;
-       description = "Port to listen for REST connections.";
+       description = mdDoc "Port to listen for REST connections.";
     };
     package = mkOption {
       type = types.package;
       default = config.nix-bitcoin.pkgs.lightning-pool;
       defaultText = "config.nix-bitcoin.pkgs.lightning-pool";
-      description = "The package providing lightning-pool binaries.";
+      description = mdDoc "The package providing lightning-pool binaries.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/lightning-pool";
-      description = "The data directory for lightning-pool.";
+      description = mdDoc "The data directory for lightning-pool.";
     };
     proxy = mkOption {
       type = types.nullOr types.str;
       default = if cfg.tor.proxy then config.nix-bitcoin.torClientAddressWithPort else null;
-      description = "host:port of SOCKS5 proxy for connnecting to the pool auction server.";
+      description = mdDoc "host:port of SOCKS5 proxy for connnecting to the pool auction server.";
     };
     extraConfig = mkOption {
       type = types.lines;
@@ -46,17 +46,17 @@ let
       example = ''
         debuglevel=trace
       '';
-      description = "Extra lines appended to the configuration file.";
+      description = mdDoc "Extra lines appended to the configuration file.";
     };
     cli = mkOption {
-      default = pkgs.writeScriptBin "pool" ''
+      default = pkgs.writers.writeBashBin "pool" ''
         exec ${cfg.package}/bin/pool \
           --rpcserver ${nbLib.addressWithPort cfg.rpcAddress cfg.rpcPort} \
           --network ${network} \
           --basedir '${cfg.dataDir}' "$@"
       '';
       defaultText = "(See source)";
-      description = "Binary to connect with the lightning-pool instance.";
+      description = mdDoc "Binary to connect with the lightning-pool instance.";
     };
     tor = nbLib.tor;
   };

modules/liquid.nix

@@ -8,19 +8,19 @@ let
       address = mkOption {
         type = types.str;
         default = "127.0.0.1";
-        description = "Address to listen for peer connections.";
+        description = mdDoc "Address to listen for peer connections.";
       };
       port = mkOption {
         type = types.port;
         default = 7042;
-        description = "Override the default port on which to listen for connections.";
+        description = mdDoc "Override the default port on which to listen for connections.";
       };
       onionPort = mkOption {
         type = types.nullOr types.port;
         # When the liquidd onion service is enabled, add an onion-tagged socket
         # to distinguish local connections from Tor connections
         default = if (config.nix-bitcoin.onionServices.liquidd.enable or false) then 7043 else null;
-        description = ''
+        description = mdDoc ''
           Port to listen for Tor peer connections.
           If set, inbound connections to this port are tagged as onion peers.
         '';
@@ -28,15 +28,15 @@ let
       listen = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = mdDoc ''
           Listen for peer connections at `address:port`
-          and `address:onionPort` (if `onionPort` is set).
+          and `address:onionPort` (if {option}`onionPort` is set).
         '';
       };
       listenWhitelisted = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = mdDoc ''
           Listen for peer connections at `address:whitelistedPort`.
           Peers connected through this socket are automatically whitelisted.
         '';
@@ -44,7 +44,7 @@ let
       whitelistedPort = mkOption {
         type = types.port;
         default = 7044;
-        description = "See `listenWhitelisted`.";
+        description = mdDoc "See {option}`listenWhitelisted`.";
       };
       extraConfig = mkOption {
         type = types.lines;
@@ -54,23 +54,23 @@ let
           rpcthreads=16
           logips=1
         '';
-        description = "Extra lines appended to <filename>elements.conf</filename>.";
+        description = mdDoc "Extra lines appended to {file}`elements.conf`.";
       };
       dataDir = mkOption {
         type = types.path;
         default = "/var/lib/liquidd";
-        description = "The data directory for liquidd.";
+        description = mdDoc "The data directory for liquidd.";
       };
       rpc = {
         address = mkOption {
           type = types.str;
           default = "127.0.0.1";
-          description = "Address to listen for JSON-RPC connections.";
+          description = mdDoc "Address to listen for JSON-RPC connections.";
         };
         port = mkOption {
           type = types.port;
           default = 7041;
-          description = "Port to listen for JSON-RPC connections.";
+          description = mdDoc "Port to listen for JSON-RPC connections.";
         };
         users = mkOption {
           default = {};
@@ -79,7 +79,7 @@ let
             bob.passwordHMAC = "b2dd077cb54591a2f3139e69a897ac$4e71f08d48b4347cf8eff3815c0e25ae2e9a4340474079f55705f40574f4ec99";
           };
           type = with types; attrsOf (submodule rpcUserOpts);
-          description = ''
+          description = mdDoc ''
             RPC user information for JSON-RPC connections.
           '';
         };
@@ -87,25 +87,25 @@ let
       rpcallowip = mkOption {
         type = types.listOf types.str;
         default = [ "127.0.0.1" ];
-        description = ''
+        description = mdDoc ''
           Allow JSON-RPC connections from specified source.
         '';
       };
       rpcuser = mkOption {
           type = types.str;
           default = "liquidrpc";
-          description = "Username for JSON-RPC connections";
+          description = mdDoc "Username for JSON-RPC connections";
       };
       proxy = mkOption {
         type = types.nullOr types.str;
         default = if cfg.tor.proxy then config.nix-bitcoin.torClientAddressWithPort else null;
-        description = "Connect through SOCKS5 proxy";
+        description = mdDoc "Connect through SOCKS5 proxy";
       };
       dbCache = mkOption {
         type = types.nullOr (types.ints.between 4 16384);
         default = null;
         example = 4000;
-        description = "Override the default database cache size in megabytes.";
+        description = mdDoc "Override the default database cache size in megabytes.";
       };
       prune = mkOption {
         type = types.nullOr (types.coercedTo
@@ -115,13 +115,13 @@ let
         );
         default = null;
         example = 10000;
-        description = ''
+        description = mdDoc ''
           Reduce storage requirements by enabling pruning (deleting) of old
           blocks. This allows the pruneblockchain RPC to be called to delete
           specific blocks, and enables automatic pruning of old blocks if a
           target size in MiB is provided. This mode is incompatible with -txindex
           and -rescan. Warning: Reverting this setting requires re-downloading
-          the entire blockchain. ("disable" = disable pruning blocks, "manual"
+          the entire blockchain. (`disable` = disable pruning blocks, `manual`
           = allow manual pruning via RPC, >=550 = automatically prune block files
           to stay under the specified target size in MiB)
         '';
@@ -129,34 +129,34 @@ let
       validatepegin = mkOption {
         type = types.nullOr types.bool;
         default = null;
-        description = ''
+        description = mdDoc ''
           Validate pegin claims. All functionaries must run this.
         '';
       };
       user = mkOption {
         type = types.str;
         default = "liquid";
-        description = "The user as which to run liquidd.";
+        description = mdDoc "The user as which to run liquidd.";
       };
       group = mkOption {
         type = types.str;
         default = cfg.user;
-        description = "The group as which to run liquidd.";
+        description = mdDoc "The group as which to run liquidd.";
       };
       cli = mkOption {
         readOnly = true;
-        default = pkgs.writeScriptBin "elements-cli" ''
+        default = pkgs.writers.writeBashBin "elements-cli" ''
           ${nbPkgs.elementsd}/bin/elements-cli -datadir='${cfg.dataDir}' "$@"
         '';
         defaultText = "(See source)";
-        description = "Binary to connect with the liquidd instance.";
+        description = mdDoc "Binary to connect with the liquidd instance.";
       };
       swapCli = mkOption {
-        default = pkgs.writeScriptBin "liquidswap-cli" ''
+        default = pkgs.writers.writeBashBin "liquidswap-cli" ''
           ${nbPkgs.liquid-swap}/bin/liquidswap-cli -c '${cfg.dataDir}/elements.conf' "$@"
         '';
         defaultText = "(See source)";
-        description = "Binary for managing liquid swaps.";
+        description = mdDoc "Binary for managing liquid swaps.";
       };
       tor = nbLib.tor;
     };
@@ -215,16 +215,16 @@ let
       name = mkOption {
         type = types.str;
         example = "alice";
-        description = ''
+        description = mdDoc ''
           Username for JSON-RPC connections.
         '';
       };
       passwordHMAC = mkOption {
         type = with types; uniq (strMatching "[0-9a-f]+\\$[0-9a-f]{64}");
         example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae";
-        description = ''
+        description = mdDoc ''
           Password HMAC-SHA-256 for JSON-RPC connections. Must be a string of the
-          format `salt-hex$hmac-hex`.
+          format `<SALT-HEX>$<HMAC-HEX>`.
         '';
       };
     };
@@ -270,8 +270,8 @@ in {
         NotifyAccess = "all";
         User = cfg.user;
         Group = cfg.group;
-        TimeoutStartSec = "10min";
-        TimeoutStopSec = "10min";
+        TimeoutStartSec = "2h";
+        TimeoutStopSec = "2h";
         ExecStart = "${nbPkgs.elementsd}/bin/elementsd -datadir='${cfg.dataDir}'";
         Restart = "on-failure";
         ReadWritePaths = [ cfg.dataDir ];

modules/lnd.nix

@@ -7,47 +7,47 @@ let
     address = mkOption {
       type = types.str;
       default = "localhost";
-      description = "Address to listen for peer connections";
+      description = mdDoc "Address to listen for peer connections";
     };
     port = mkOption {
       type = types.port;
       default = 9735;
-      description = "Port to listen for peer connections";
+      description = mdDoc "Port to listen for peer connections";
     };
     rpcAddress = mkOption {
       type = types.str;
       default = "localhost";
-      description = "Address to listen for RPC connections.";
+      description = mdDoc "Address to listen for RPC connections.";
     };
     rpcPort = mkOption {
       type = types.port;
       default = 10009;
-      description = "Port to listen for gRPC connections.";
+      description = mdDoc "Port to listen for gRPC connections.";
     };
     restAddress = mkOption {
       type = types.str;
       default = "localhost";
-      description = "Address to listen for REST connections.";
+      description = mdDoc "Address to listen for REST connections.";
     };
     restPort = mkOption {
       type = types.port;
       default = 8080;
-      description = "Port to listen for REST connections.";
+      description = mdDoc "Port to listen for REST connections.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/lnd";
-      description = "The data directory for LND.";
+      description = mdDoc "The data directory for LND.";
     };
     networkDir = mkOption {
       readOnly = true;
       default = "${cfg.dataDir}/chain/bitcoin/${bitcoind.network}";
-      description = "The network data directory.";
+      description = mdDoc "The network data directory.";
     };
     tor-socks = mkOption {
       type = types.nullOr types.str;
       default = if cfg.tor.proxy then config.nix-bitcoin.torClientAddressWithPort else null;
-      description = "Socks proxy for connecting to Tor nodes";
+      description = mdDoc "Socks proxy for connecting to Tor nodes";
     };
     macaroons = mkOption {
       default = {};
@@ -55,18 +55,18 @@ let
         options = {
           user = mkOption {
             type = types.str;
-            description = "User who owns the macaroon.";
+            description = mdDoc "User who owns the macaroon.";
           };
           permissions = mkOption {
             type = types.str;
             example = ''
               {"entity":"info","action":"read"},{"entity":"onchain","action":"read"}
             '';
-            description = "List of granted macaroon permissions.";
+            description = mdDoc "List of granted macaroon permissions.";
           };
         };
       });
-      description = ''
+      description = mdDoc ''
         Extra macaroon definitions.
       '';
     };
@@ -75,18 +75,18 @@ let
         type = with types; listOf str;
         default = [];
         example = [ "60.100.0.1" ];
-        description = ''
+        description = mdDoc ''
           Extra `subjectAltName` IPs added to the certificate.
-          This works the same as lnd option `tlsextraip`.
+          This works the same as lnd option {option}`tlsextraip`.
         '';
       };
       extraDomains = mkOption {
         type = with types; listOf str;
         default = [];
         example = [ "example.com" ];
-        description = ''
+        description = mdDoc ''
           Extra `subjectAltName` domain names added to the certificate.
-          This works the same as lnd option `tlsextradomain`.
+          This works the same as lnd option {option}`tlsextradomain`.
         '';
       };
     };
@@ -96,8 +96,8 @@ let
       example = ''
         autopilot.active=1
       '';
-      description = ''
-        Extra lines appended to `lnd.conf`.
+      description = mdDoc ''
+        Extra lines appended to {file}`lnd.conf`.
         See here for all available options:
         https://github.com/lightningnetwork/lnd/blob/master/sample-lnd.conf
       '';
@@ -106,10 +106,10 @@ let
       type = types.package;
       default = config.nix-bitcoin.pkgs.lnd;
       defaultText = "config.nix-bitcoin.pkgs.lnd";
-      description = "The package providing lnd binaries.";
+      description = mdDoc "The package providing lnd binaries.";
     };
     cli = mkOption {
-      default = pkgs.writeScriptBin "lncli"
+      default = pkgs.writers.writeBashBin "lncli"
         # Switch user because lnd makes datadir contents readable by user only
         ''
           ${runAsUser} ${cfg.user} ${cfg.package}/bin/lncli \
@@ -118,12 +118,12 @@ let
             --macaroonpath '${networkDir}/admin.macaroon' "$@"
         '';
       defaultText = "(See source)";
-      description = "Binary to connect with the lnd instance.";
+      description = mdDoc "Binary to connect with the lnd instance.";
     };
     getPublicAddressCmd = mkOption {
       type = types.str;
       default = "";
-      description = ''
+      description = mdDoc ''
         Bash expression which outputs the public service address to announce to peers.
         If left empty, no address is announced.
       '';
@@ -131,17 +131,17 @@ let
     user = mkOption {
       type = types.str;
       default = "lnd";
-      description = "The user as which to run LND.";
+      description = mdDoc "The user as which to run LND.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run LND.";
+      description = mdDoc "The group as which to run LND.";
     };
     certPath = mkOption {
       readOnly = true;
       default = "${secretsDir}/lnd-cert";
-      description = "LND TLS certificate path.";
+      description = mdDoc "LND TLS certificate path.";
     };
     tor = nbLib.tor;
   };
@@ -174,19 +174,26 @@ let
     ${optionalString (cfg.tor-socks != null) "tor.socks=${cfg.tor-socks}"}
 
     bitcoind.rpchost=${bitcoindRpcAddress}:${toString bitcoind.rpc.port}
-    bitcoind.rpcuser=${bitcoind.rpc.users.public.name}
-    bitcoind.zmqpubrawblock=${bitcoind.zmqpubrawblock}
-    bitcoind.zmqpubrawtx=${bitcoind.zmqpubrawtx}
+    bitcoind.rpcuser=${bitcoind.rpc.users.${rpcUser}.name}
+    bitcoind.zmqpubrawblock=${zmqHandleSpecialAddress bitcoind.zmqpubrawblock}
+    bitcoind.zmqpubrawtx=${zmqHandleSpecialAddress bitcoind.zmqpubrawtx}
 
     wallet-unlock-password-file=${secretsDir}/lnd-wallet-password
 
     ${cfg.extraConfig}
   '';
+
+  zmqHandleSpecialAddress = builtins.replaceStrings [ "0.0.0.0" "[::]" ] [ "127.0.0.1" "[::1]" ];
+
+  isPruned = bitcoind.prune > 0;
+  # When bitcoind pruning is enabled, lnd requires non-public RPC commands `getpeerinfo`, `getnodeaddresses`
+  # to fetch missing blocks from peers (implemented in btcsuite/btcwallet/chain/pruned_block_dispatcher.go)
+  rpcUser = if isPruned then "lnd" else "public";
 in {
 
   inherit options;
 
-  config = mkIf cfg.enable {
+  config = mkIf cfg.enable (mkMerge [ {
     assertions = [
       { assertion =
           !(config.services ? clightning)
@@ -207,8 +214,8 @@ in {
       # under high bitcoind rpc load
       rpc.threads = 16;
 
-      zmqpubrawblock = "tcp://${bitcoindRpcAddress}:28332";
-      zmqpubrawtx = "tcp://${bitcoindRpcAddress}:28333";
+      zmqpubrawblock = mkDefault "tcp://${bitcoindRpcAddress}:28332";
+      zmqpubrawtx = mkDefault "tcp://${bitcoindRpcAddress}:28333";
     };
 
     environment.systemPackages = [ cfg.package (hiPrio cfg.cli) ];
@@ -226,7 +233,7 @@ in {
       preStart = ''
         install -m600 ${configFile} '${cfg.dataDir}/lnd.conf'
         {
-          echo "bitcoind.rpcpass=$(cat ${secretsDir}/bitcoin-rpcpassword-public)"
+          echo "bitcoind.rpcpass=$(cat ${secretsDir}/bitcoin-rpcpassword-${rpcUser})"
           ${optionalString (cfg.getPublicAddressCmd != "") ''
             echo "externalip=$(${cfg.getPublicAddressCmd})"
           ''}
@@ -304,5 +311,22 @@ in {
       makePasswordSecret lnd-wallet-password
       makeCert lnd '${nbLib.mkCertExtraAltNames cfg.certificate}'
     '';
-  };
+  }
+
+  (mkIf isPruned {
+    services.bitcoind.rpc.users.lnd = {
+      passwordHMACFromFile = true;
+      rpcwhitelist = bitcoind.rpc.users.public.rpcwhitelist ++ [
+        "getpeerinfo"
+        "getnodeaddresses"
+      ];
+    };
+    nix-bitcoin.secrets = {
+      bitcoin-rpcpassword-lnd.user = cfg.user;
+      bitcoin-HMAC-lnd.user = bitcoind.user;
+    };
+    nix-bitcoin.generateSecretsCmds.lndBitcoinRPC = ''
+      makeBitcoinRPCPassword lnd
+    '';
+  }) ]);
 }

modules/lndconnect-onion.nix

@@ -1,126 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-let
-  options = {
-    services.lnd.lndconnectOnion.enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Create an onion service for the lnd REST server.
-        Add a `lndconnect-onion` binary to the system environment.
-        See: https://github.com/LN-Zap/lndconnect
-
-        Usage:
-        ```
-          # Print QR code
-          lndconnect-onion
-
-          # Print URL
-          lndconnect-onion --url
-        ```
-      '';
-    };
-
-    services.clightning-rest.lndconnectOnion.enable = mkOption {
-      type = types.bool;
-      default = false;
-      description = ''
-        Create an onion service for clightning-rest.
-        Add a `lndconnect-onion-clightning` binary to the system environment.
-        See: https://github.com/LN-Zap/lndconnect
-
-        Usage:
-        ```
-          # Print QR code
-          lndconnect-onion-clightning
-
-          # Print URL
-          lndconnect-onion-clightning --url
-        ```
-      '';
-    };
-  };
-
-  nbLib = config.nix-bitcoin.lib;
-  runAsUser = config.nix-bitcoin.runAsUserCmd;
-
-  inherit (config.services)
-    lnd
-    clightning
-    clightning-rest;
-
-  mkLndconnect = {
-    name,
-    shebang ? "#!${pkgs.stdenv.shell} -e",
-    onionService,
-    port,
-    certPath,
-    macaroonPath
-  }:
-  # TODO-EXTERNAL:
-  # lndconnect requires a --configfile argument, although it's unused
-  # https://github.com/LN-Zap/lndconnect/issues/25
-  pkgs.writeScriptBin name ''
-    ${shebang}
-    exec ${config.nix-bitcoin.pkgs.lndconnect}/bin/lndconnect \
-     --host=$(cat ${config.nix-bitcoin.onionAddresses.dataDir}/${onionService}) \
-     --port=${toString port} \
-     --tlscertpath='${certPath}' \
-     --adminmacaroonpath='${macaroonPath}' \
-     --configfile=/dev/null "$@"
-  '';
-
-  operatorName = config.nix-bitcoin.operator.name;
-in {
-  inherit options;
-
-  config = mkMerge [
-    (mkIf (lnd.enable && lnd.lndconnectOnion.enable) {
-      services.tor = {
-        enable = true;
-        relay.onionServices.lnd-rest = nbLib.mkOnionService {
-          target.addr = nbLib.address lnd.restAddress;
-          target.port = lnd.restPort;
-          port = lnd.restPort;
-        };
-      };
-      nix-bitcoin.onionAddresses.access.${lnd.user} = [ "lnd-rest" ];
-
-      environment.systemPackages = [(
-        mkLndconnect {
-          name = "lndconnect-onion";
-          # Run as lnd user because the macaroon and cert are not group-readable
-          shebang = "#!/usr/bin/env -S ${runAsUser} ${lnd.user} ${pkgs.bash}/bin/bash";
-          onionService = "${lnd.user}/lnd-rest";
-          port = lnd.restPort;
-          certPath = lnd.certPath;
-          macaroonPath = "${lnd.networkDir}/admin.macaroon";
-        }
-      )];
-    })
-
-    (mkIf (clightning-rest.enable && clightning-rest.lndconnectOnion.enable) {
-      services.tor = {
-        enable = true;
-        relay.onionServices.clightning-rest = nbLib.mkOnionService {
-          target.addr = nbLib.address clightning-rest.address;
-          target.port = clightning-rest.port;
-          port = clightning-rest.port;
-        };
-      };
-      # This also allows nodeinfo to show the clightning-rest onion address
-      nix-bitcoin.onionAddresses.access.${operatorName} = [ "clightning-rest" ];
-
-      environment.systemPackages = [(
-        mkLndconnect {
-          name = "lndconnect-onion-clightning";
-          onionService = "${operatorName}/clightning-rest";
-          port = clightning-rest.port;
-          certPath = "${clightning-rest.dataDir}/certs/certificate.pem";
-          macaroonPath = "${clightning-rest.dataDir}/certs/access.macaroon";
-        }
-      )];
-    })
-  ];
-}

modules/lndconnect.nix

@@ -0,0 +1,205 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  options = {
+    services.lnd.lndconnect = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = mdDoc ''
+          Add a `lndconnect` binary to the system environment which prints
+          connection info for lnd clients.
+          See: https://github.com/LN-Zap/lndconnect
+
+          Usage:
+          ```bash
+            # Print QR code
+            lndconnect
+
+            # Print URL
+            lndconnect --url
+          ```
+        '';
+      };
+      onion = mkOption {
+        type = types.bool;
+        default = false;
+        description = mdDoc ''
+          Create an onion service for the lnd REST server,
+          which is used by lndconnect.
+        '';
+      };
+    };
+
+
+    services.clightning-rest.lndconnect = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = mdDoc ''
+        Add a `lndconnect-clightning` binary to the system environment which prints
+        connection info for clightning clients.
+        See: https://github.com/LN-Zap/lndconnect
+
+        Usage:
+        ```bash
+          # Print QR code
+          lndconnect-clightning
+
+          # Print URL
+          lndconnect-clightning --url
+        ```
+      '';
+      };
+      onion = mkOption {
+        type = types.bool;
+        default = false;
+        description = mdDoc ''
+          Create an onion service for the clightning REST server,
+          which is used by lndconnect.
+        '';
+      };
+    };
+
+    nix-bitcoin.mkLndconnect = mkOption {
+      readOnly = true;
+      default = mkLndconnect;
+      description = mdDoc ''
+        A function to create a lndconnect binary.
+        See the source for further details.
+      '';
+    };
+  };
+
+  nbLib = config.nix-bitcoin.lib;
+  runAsUser = config.nix-bitcoin.runAsUserCmd;
+
+  inherit (config.services)
+    lnd
+    clightning-rest;
+
+  mkLndconnect = {
+    name,
+    shebang ? "#!${pkgs.stdenv.shell} -e",
+    isClightning ? false,
+    port,
+    macaroonPath,
+    enableOnion,
+    onionService ? null,
+    certPath ? null
+  }:
+  # TODO-EXTERNAL:
+  # lndconnect requires a --configfile argument, although it's unused
+  # https://github.com/LN-Zap/lndconnect/issues/25
+  pkgs.hiPrio (pkgs.writeScriptBin name ''
+    ${shebang}
+    url=$(
+      ${getExe config.nix-bitcoin.pkgs.lndconnect} --url \
+        ${optionalString enableOnion "--host=$(cat ${config.nix-bitcoin.onionAddresses.dataDir}/${onionService})"} \
+        --port=${toString port} \
+        ${if enableOnion || certPath == null then "--nocert" else "--tlscertpath='${certPath}'"} \
+        --adminmacaroonpath='${macaroonPath}' \
+        --configfile=/dev/null "$@"
+    )
+
+    ${optionalString isClightning
+      # - Change URL procotcol to c-lightning-rest
+      # - Encode macaroon as hex (in uppercase) instead of base 64.
+      #   Because `macaroon` is always the last URL fragment, the
+      #   sed replacement below works correctly.
+      ''
+        macaroonHex=$(${getExe pkgs.xxd} -p -u -c 99999 '${macaroonPath}')
+        url=$(
+          echo "$url" | ${getExe pkgs.gnused} "
+            s|^lndconnect|c-lightning-rest|
+            s|macaroon=.*|macaroon=$macaroonHex|
+          ";
+        )
+      ''
+    }
+
+    # If --url is in args
+    if [[ " $* " =~ " --url " ]]; then
+      echo "$url"
+    else
+      # This UTF-8 encoding yields a smaller, more convenient output format
+      # compared to the native lndconnect output
+      echo -n "$url" | ${getExe pkgs.qrencode} -t UTF8 -o -
+    fi
+  '');
+
+  operatorName = config.nix-bitcoin.operator.name;
+in {
+  inherit options;
+
+  config = mkMerge [
+    (mkIf (lnd.enable && lnd.lndconnect.enable)
+      (mkMerge [
+        {
+          environment.systemPackages = [(
+            mkLndconnect {
+              name = "lndconnect";
+              # Run as lnd user because the macaroon and cert are not group-readable
+              shebang = "#!/usr/bin/env -S ${runAsUser} ${lnd.user} ${pkgs.bash}/bin/bash";
+              enableOnion = lnd.lndconnect.onion;
+              onionService = "${lnd.user}/lnd-rest";
+              port = lnd.restPort;
+              certPath = lnd.certPath;
+              macaroonPath = "${lnd.networkDir}/admin.macaroon";
+            }
+          )];
+
+          services.lnd.restAddress = mkIf (!lnd.lndconnect.onion) "0.0.0.0";
+        }
+
+        (mkIf lnd.lndconnect.onion {
+          services.tor = {
+            enable = true;
+            relay.onionServices.lnd-rest = nbLib.mkOnionService {
+              target.addr = nbLib.address lnd.restAddress;
+              target.port = lnd.restPort;
+              port = lnd.restPort;
+            };
+          };
+          nix-bitcoin.onionAddresses.access = {
+            ${lnd.user} = [ "lnd-rest" ];
+            ${operatorName} = [ "lnd-rest" ];
+          };
+        })
+      ]))
+
+    (mkIf (clightning-rest.enable && clightning-rest.lndconnect.enable)
+      (mkMerge [
+        {
+          environment.systemPackages = [(
+            mkLndconnect {
+              name = "lndconnect-clightning";
+              isClightning = true;
+              enableOnion = clightning-rest.lndconnect.onion;
+              onionService = "${operatorName}/clightning-rest";
+              port = clightning-rest.port;
+              certPath = "${clightning-rest.dataDir}/certs/certificate.pem";
+              macaroonPath = "${clightning-rest.dataDir}/certs/access.macaroon";
+            }
+          )];
+
+          # clightning-rest always binds to all interfaces
+        }
+
+        (mkIf clightning-rest.lndconnect.onion {
+          services.tor = {
+            enable = true;
+            relay.onionServices.clightning-rest = nbLib.mkOnionService {
+              target.addr = nbLib.address clightning-rest.address;
+              target.port = clightning-rest.port;
+              port = clightning-rest.port;
+            };
+          };
+          # This also allows nodeinfo to show the clightning-rest onion address
+          nix-bitcoin.onionAddresses.access.${operatorName} = [ "clightning-rest" ];
+        })
+      ])
+    )
+  ];
+}

modules/modules.nix

@@ -19,7 +19,7 @@
     ./lightning-loop.nix
     ./lightning-pool.nix
     ./charge-lnd.nix
-    ./lndconnect-onion.nix # Requires onion-addresses.nix
+    ./lndconnect.nix # Requires onion-addresses.nix
     ./rtl.nix
     ./electrs.nix
     ./fulcrum.nix

modules/netns-isolation.nix

@@ -8,7 +8,7 @@ let
     addressblock = mkOption {
       type = types.ints.u8;
       default = 1;
-      description = ''
+      description = mdDoc ''
         The address block N in 169.254.N.0/24, used as the prefix for netns addresses.
       '';
     };
@@ -20,7 +20,7 @@ let
           id = mkOption {
             # TODO: Assert uniqueness
             type = types.ints.between 11 255;
-            description = ''
+            description = mdDoc ''
               id for the netns, used for the IP address host part and
               for naming the interfaces. Must be unique. Must be greater than 10.
             '';
@@ -35,7 +35,7 @@ let
 
     allowedUser = mkOption {
       type = types.str;
-      description = ''
+      description = mdDoc ''
         User that is allowed to execute commands in the service network namespaces.
         The user's group is also authorized.
       '';
@@ -45,13 +45,13 @@ let
     netns = mkOption {
       readOnly = true;
       default = netns;
-      description = "Exposes netns parameters.";
+      description = mdDoc "Exposes netns parameters.";
     };
 
     bridgeIp = mkOption {
       readOnly = true;
       default = bridgeIp;
-      description = "IP of the netns bridge interface.";
+      description = mdDoc "IP of the netns bridge interface.";
     };
   };
 
@@ -215,9 +215,11 @@ in {
 
         };
       };
-    in foldl (services: n:
+    in
+      foldl (services: n:
         services // (makeNetnsServices n netns.${n})
-    ) {} (builtins.attrNames netns));
+      ) {} (builtins.attrNames netns)
+    );
   }
 
   # Service-specific config
@@ -297,6 +299,7 @@ in {
         id = 31;
         connections = [ "bitcoind" ];
       };
+      # id = 32 reserved for the upcoming mempool module
     };
 
     services.bitcoind = {

modules/nix-bitcoin.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, options, pkgs, lib, ... }:
 
 with lib;
 {
@@ -8,11 +8,23 @@ with lib;
         type = types.attrs;
         default = (import ../pkgs { inherit pkgs; }).modulesPkgs;
         defaultText = "nix-bitcoin/pkgs.modulesPkgs";
+        apply = base:
+          let
+            final = foldl (prev: overlay:
+              prev // (overlay prev final)
+            ) base options.nix-bitcoin.pkgOverlays.definitions;
+          in
+            final;
+      };
+
+      pkgOverlays = mkOption {
+        internal = true;
+        type = with types; functionTo attrs;
       };
 
       lib = mkOption {
         readOnly = true;
-        default = import ../pkgs/lib.nix lib pkgs;
+        default = import ../pkgs/lib.nix lib pkgs config;
         defaultText = "nix-bitcoin/pkgs/lib.nix";
       };
 
@@ -27,7 +39,7 @@ with lib;
       # Related issue: https://github.com/NixOS/nixpkgs/issues/94236
       torify = mkOption {
         readOnly = true;
-        default = pkgs.writeScriptBin "torify" ''
+        default = pkgs.writers.writeBashBin "torify" ''
           ${pkgs.tor}/bin/torify \
             --address ${config.services.tor.client.socksListenAddress.addr} \
             "$@"
@@ -39,8 +51,7 @@ with lib;
       runAsUserCmd = mkOption {
         readOnly = true;
         default = if config.security.doas.enable
-                  # TODO-EXTERNAL: Use absolute path until https://github.com/NixOS/nixpkgs/pull/133622 is available.
-                  then "/run/wrappers/bin/doas -u"
+                  then "doas -u"
                   else "sudo -u";
         defaultText = "(See source)";
       };

modules/nodeinfo.nix

@@ -17,17 +17,17 @@ let
         type = types.attrs;
         default = {};
         defaultText = "(See source)";
-        description = ''
+        description = mdDoc ''
           Nodeinfo service definitions.
         '';
       };
 
-      nodeinfoLib = mkOption {
+      lib = mkOption {
         internal = true;
         readOnly = true;
         default = nodeinfoLib;
         defaultText = "(See source)";
-        description = ''
+        description = mdDoc ''
           Helper functions for defining nodeinfo services.
         '';
       };
@@ -63,7 +63,7 @@ let
     infos = OrderedDict()
     operator = "${config.nix-bitcoin.operator.name}"
 
-    def set_onion_address(info, name, port):
+    def get_onion_address(name, port):
         path = f"/var/lib/onion-addresses/{operator}/{name}"
         try:
             with open(path, "r") as f:
@@ -71,11 +71,12 @@ let
         except OSError:
             print(f"error reading file {path}", file=sys.stderr)
             return
-        info["onion_address"] = f"{onion_address}:{port}"
+        return f"{onion_address}:{port}"
 
-    def add_service(service, make_info):
-        if not is_active(service):
-            infos[service] = "service is not running"
+    def add_service(service, make_info, systemd_service = None):
+        systemd_service = systemd_service or service
+        if not is_active(systemd_service):
+            infos[service] = f"'{systemd_service}.service' is not running"
         else:
             info = OrderedDict()
             exec(make_info, globals(), locals())
@@ -96,14 +97,19 @@ let
   ) (builtins.attrNames cfg.services);
 
   nodeinfoLib = rec {
-    mkInfo = extraCode: name: cfg: ''
+    mkInfo = extraCode: name: cfg:
+      mkInfoLong {
+        inherit extraCode name cfg;
+      };
+
+    mkInfoLong = { extraCode ? "", name, cfg, systemdServiceName ? name }: ''
       add_service("${name}", """
       info["local_address"] = "${nbLib.addressWithPort cfg.address cfg.port}"
     '' + mkIfOnionPort name (onionPort: ''
-      set_onion_address(info, "${name}", ${onionPort})
+      info["onion_address"] = get_onion_address("${name}", ${onionPort})
     '') + extraCode + ''
 
-      """)
+      """, "${systemdServiceName}")
     '';
 
     mkIfOnionPort = name: fn:
@@ -117,8 +123,10 @@ let
 in {
   inherit options;
 
-  config = {
-    environment.systemPackages = optional cfg.enable script;
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ script ];
+
+    nix-bitcoin.operator.enable = true;
 
     nix-bitcoin.nodeinfo.services = with nodeinfoLib; {
       bitcoind = mkInfo "";
@@ -127,9 +135,13 @@ in {
         if 'onion_address' in info:
             info["id"] = f"{info['nodeid']}@{info['onion_address']}"
       '';
-      lnd = mkInfo ''
+      lnd = name: cfg: mkInfo (''
+        info["rest_address"] = "${nbLib.addressWithPort cfg.restAddress cfg.restPort}"
+      '' + mkIfOnionPort "lnd-rest" (onionPort: ''
+        info["onion_rest_address"] = get_onion_address("lnd-rest", ${onionPort})
+      '') + ''
         info["nodeid"] = shell("lncli getinfo | jq -r '.identity_pubkey'")
-      '';
+      '') name cfg;
       clightning-rest = mkInfo "";
       electrs = mkInfo "";
       fulcrum = mkInfo "";
@@ -140,7 +152,7 @@ in {
       rtl = mkInfo "";
       # Only add sshd when it has an onion service
       sshd = name: cfg: mkIfOnionPort "sshd" (onionPort: ''
-        add_service("sshd", """set_onion_address(info, "sshd", ${onionPort})""")
+        add_service("sshd", """info["onion_address"] = get_onion_address("sshd", ${onionPort})""")
       '');
     };
   };

modules/obsolete-options.nix

@@ -33,7 +33,6 @@ in {
     (mkRenamedOptionModule [ "services" "liquidd" "rpcbind" ] [ "services" "liquidd" "rpc" "address" ])
     # 0.0.70
     (mkRenamedOptionModule [ "services" "rtl" "cl-rest" ] [ "services" "clightning-rest" ])
-    (mkRenamedOptionModule [ "services" "lnd" "restOnionService" "enable" ] [ "services" "lnd" "lndconnectOnion" "enable" ])
 
     (mkRenamedOptionModule [ "nix-bitcoin" "setup-secrets" ] [ "nix-bitcoin" "setupSecrets" ])
 
@@ -46,6 +45,28 @@ in {
       bitcoin peer connections for syncing blocks. This performs well on low and high
       memory systems.
     '')
+    # 0.0.86
+    (mkRemovedOptionModule [ "services" "lnd" "restOnionService" "enable" ] ''
+      Set the following options instead:
+      services.lnd.lndconnect = {
+        enable = true;
+        onion = true;
+      }
+    '')
+    (mkRemovedOptionModule [ "services" "lnd" "lndconnectOnion" ] ''
+      Set the following options instead:
+      services.lnd.lndconnect = {
+        enable = true;
+        onion = true;
+      }
+    '')
+    (mkRemovedOptionModule [ "services" "clightning-rest" "lndconnectOnion" ] ''
+      Set the following options instead:
+      services.clightning-rest.lndconnect = {
+        enable = true;
+        onion = true;
+      }
+    '')
   ] ++
   # 0.0.59
   (map mkSplitEnforceTorOption [

modules/onion-addresses.nix

@@ -12,23 +12,25 @@ let
     access = mkOption {
       type = with types; attrsOf (listOf str);
       default = {};
-      description = ''
+      description = mdDoc ''
         This option controls who is allowed to access onion addresses.
         For example, the following allows user 'myuser' to access bitcoind
         and clightning onion addresses:
+        ```nix
         {
           "myuser" = [ "bitcoind" "clightning" ];
         };
+        ```
         The onion hostnames can then be read from
-        /var/lib/onion-addresses/myuser.
+        {file}`/var/lib/onion-addresses/myuser`.
       '';
     };
     services = mkOption {
       type = with types; listOf str;
       default = [];
-      description = ''
+      description = mdDoc ''
         Services that can access their onion address via file
-        `/var/lib/onion-addresses/$service`
+        {file}`/var/lib/onion-addresses/<service>`
         The file is readable only by the service user.
       '';
     };
@@ -53,8 +55,8 @@ in {
         RemainAfterExit = true;
         StateDirectory = "onion-addresses";
         StateDirectoryMode = "771";
-        PrivateNetwork = "true"; # This service needs no network access
-        PrivateUsers = "false";
+        PrivateNetwork = true; # This service needs no network access
+        PrivateUsers = false;
         CapabilityBoundingSet = "CAP_CHOWN CAP_FSETID CAP_SETFCAP CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_IPC_OWNER";
       };
       script = ''

modules/onion-services.nix

@@ -16,24 +16,24 @@ let
           enable = mkOption {
             type = types.bool;
             default = config.public;
-            description = ''
+            description = mdDoc ''
               Create an onion service for the given service.
-              The service must define options 'address' and 'onionPort' (or `port`).
+              The service must define options {option}'address' and {option}'onionPort' (or `port`).
             '';
           };
           public = mkOption {
             type = types.bool;
             default = false;
-            description = ''
+            description = mdDoc ''
               Make the onion address accessible to the service.
               If enabled, the onion service is automatically enabled.
-              Only available for services that define option `getPublicAddressCmd`.
+              Only available for services that define option {option}`getPublicAddressCmd`.
             '';
           };
           externalPort = mkOption {
             type = types.nullOr types.port;
             default = null;
-            description = "Override the external port of the onion service.";
+            description = mdDoc "Override the external port of the onion service.";
           };
         };
       }

modules/operator.nix

@@ -6,7 +6,7 @@ let
     enable = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Whether to define a user named `operator` for convenient interactive access
         to nix-bitcoin features (like `bitcoin-cli`).
 
@@ -18,17 +18,17 @@ let
     name = mkOption {
       type = types.str;
       default = "operator";
-      description = "Name of the operator user.";
+      description = mdDoc "Name of the operator user.";
     };
     groups = mkOption {
       type = with types; listOf str;
       default = [];
-      description = "Extra groups of the operatur user.";
+      description = mdDoc "Extra groups of the operatur user.";
     };
     allowRunAsUsers = mkOption {
       type = with types; listOf str;
       default = [];
-      description = "Users as which the operator is allowed to run commands.";
+      description = mdDoc "Users as which the operator is allowed to run commands.";
     };
   };
 

modules/presets/bitcoind-remote.nix

@@ -5,7 +5,9 @@ let
   secretsDir = config.nix-bitcoin.secretsDir;
 in {
   services.bitcoind = {
-    # Make the local bitcoin-cli work with the remote node
+    # Make the local bitcoin-cli work with the remote node.
+    # Without this, bitcoin-cli would try to use the .cookie file in the local
+    # bitcoind data dir for authorization, which doesn't exist.
     extraConfig = ''
       rpcuser=${cfg.rpc.users.privileged.name}
     '';

modules/presets/hardened.nix

@@ -1,7 +1,8 @@
-{
+{ modulesPath, ... }: {
   imports = [
-    # Source: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix
-    <nixpkgs/nixos/modules/profiles/hardened.nix>
+    # Source:
+    # https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix
+    (modulesPath + "/profiles/hardened.nix")
   ];
 
   ## Reset some options set by the hardened profile

modules/presets/wireguard.nix

@@ -0,0 +1,214 @@
+{ config, pkgs, lib, ... }:
+
+# Create a WireGuard server with a single peer.
+# Private/public keys are created via the secrets system.
+# Add helper binaries `nix-bitcoin-wg-connect` and optionally `lndconnect-wg`, `lndconnect-clightning-wg`.
+
+# See ../../docs/services.md ("Use Zeus (mobile lightning wallet) via WireGuard")
+# for usage instructions.
+
+# This is a rather opinionated implementation that lacks the flexibility offered by
+# other nix-bitcoin modules, so ship this as a `preset`.
+# Some users will prefer to use `lndconnect` with their existing WireGuard or Tailscale setup.
+
+with lib;
+let
+  options.nix-bitcoin.wireguard = {
+    subnet = mkOption {
+      type = types.str;
+      default = "10.10.0";
+      description = mdDoc "The /24 subnet of the wireguard network.";
+    };
+    restrictPeer = mkOption {
+      type = types.bool;
+      default = true;
+      description = mdDoc ''
+        Prevent the peer from connecting to any addresses except for the WireGuard server address.
+      '';
+    };
+  };
+
+  cfg = config.nix-bitcoin.wireguard;
+  wgSubnet = cfg.subnet;
+  inherit (config.networking.wireguard.interfaces) wg-nb;
+  inherit (config.services)
+    lnd
+    clightning-rest;
+
+  lndconnect = lnd.enable && lnd.lndconnect.enable;
+  lndconnect-clightning = clightning-rest.enable && clightning-rest.lndconnect.enable;
+
+  serverAddress = "${wgSubnet}.1";
+  peerAddress = "${wgSubnet}.2";
+
+  secretsDir = config.nix-bitcoin.secretsDir;
+
+  wgConnectUser = if config.nix-bitcoin.operator.enable
+                  then config.nix-bitcoin.operator.name
+                  else "root";
+
+  # A script that prints a QR code to connect a peer to the server.
+  # The QR code encodes a wg-quick config that can be imported by the wireguard
+  # mobile app.
+  wgConnect = pkgs.writers.writeBashBin "nix-bitcoin-wg-connect" ''
+    set -euo pipefail
+    text=
+    host=
+    for arg in "$@"; do
+      case $arg in
+        --text)
+          text=1
+          ;;
+        *)
+          host=$arg
+          ;;
+      esac
+    done
+
+    if [[ ! $host ]]; then
+      # Use lndconnect to fetch the external ip.
+      # This internally uses https://github.com/GlenDC/go-external-ip, which
+      # queries a set of external ip providers.
+      host=$(
+        ${getExe config.nix-bitcoin.pkgs.lndconnect} --url --nocert \
+          --configfile=/dev/null --adminmacaroonpath=/dev/null \
+          | sed -nE 's|.*?/(.*?):.*|\1|p'
+      )
+    fi
+
+    config="[Interface]
+    PrivateKey = $(cat ${secretsDir}/wg-peer-private-key)
+    Address = ${peerAddress}/24
+
+    [Peer]
+    PublicKey = $(cat ${secretsDir}/wg-server-public-key)
+    AllowedIPs = ${wgSubnet}.0/24
+    Endpoint = $host:${toString wg-nb.listenPort}
+    PersistentKeepalive = 25
+    "
+
+    if [[ $text ]]; then
+      echo "$config"
+    else
+      echo "$config" | ${getExe pkgs.qrencode} -t UTF8 -o -
+    fi
+  '';
+in {
+  inherit options;
+
+  config = {
+    assertions = [
+      {
+        # Don't support `netns-isolation` for now to keep things simple
+        assertion = !(config.nix-bitcoin.netns-isolation.enable or false);
+        message = "`nix-bitcoin.wireguard` is not compatible with `netns-isolation`.";
+      }
+    ];
+
+    networking.wireguard.interfaces.wg-nb = {
+      ips = [ "${serverAddress}/24" ];
+      listenPort = mkDefault 51820;
+      privateKeyFile = "${secretsDir}/wg-server-private-key";
+      allowedIPsAsRoutes = false;
+      peers = [
+        {
+          # To use the actual public key from the secrets file, use dummy pubkey
+          # `peer0` and replace it via `getPubkeyFromFile` (see further below)
+          # at peer service runtime.
+          publicKey = "peer0";
+          allowedIPs = [ "${peerAddress}/32" ];
+        }
+      ];
+    };
+
+    systemd.services = {
+      wireguard-wg-nb = rec {
+        wants = [ "nix-bitcoin-secrets.target" ];
+        after = wants;
+      };
+
+      # HACK: Modify start/stop scripts of the peer setup service to read
+      # the pubkey from a secrets file.
+      wireguard-wg-nb-peer-peer0 = let
+        getPubkeyFromFile = mkBefore ''
+          if [[ ! -v inPatchedSrc ]]; then
+            export inPatchedSrc=1
+            publicKey=$(cat "${secretsDir}/wg-peer-public-key")
+            <"''${BASH_SOURCE[0]}" sed "s|\bpeer0\b|$publicKey|g" | ${pkgs.bash}/bin/bash -s
+            exit
+          fi
+        '';
+      in {
+        script = getPubkeyFromFile;
+        postStop = getPubkeyFromFile;
+      };
+    };
+
+    environment.systemPackages = [
+      wgConnect
+    ] ++ (optional lndconnect
+      (pkgs.writers.writeBashBin "lndconnect-wg" ''
+        exec lndconnect --host "${serverAddress}" --nocert "$@"
+      '')
+    ) ++ (optional lndconnect-clightning
+      (pkgs.writers.writeBashBin "lndconnect-clightning-wg" ''
+        exec lndconnect-clightning --host "${serverAddress}" --nocert "$@"
+      '')
+    );
+
+    networking.firewall = let
+      restrictPeerRule = "-s ${peerAddress} ! -d ${serverAddress} -j REJECT";
+    in {
+      allowedUDPPorts = [ wg-nb.listenPort ];
+
+      extraCommands =
+        optionalString lndconnect ''
+          iptables -w -A nixos-fw -p tcp -s ${wgSubnet}.0/24 --dport ${toString lnd.restPort} -j nixos-fw-accept
+        ''
+        + optionalString lndconnect-clightning ''
+          iptables -w -A nixos-fw -p tcp -s ${wgSubnet}.0/24 --dport ${toString clightning-rest.port} -j nixos-fw-accept
+        ''
+        + optionalString cfg.restrictPeer ''
+          iptables -w -A nixos-fw ${restrictPeerRule}
+          iptables -w -A FORWARD ${restrictPeerRule}
+        '';
+
+      extraStopCommands =
+        # Rules added to chain `nixos-fw` are automatically removed when restarting
+        # the NixOS firewall service.
+        mkIf cfg.restrictPeer ''
+          iptables -w -D FORWARD ${restrictPeerRule} || :
+        '';
+    };
+
+    # Listen on all addresses, including `serverAddress`.
+    # This is safe because the listen ports are secured by the firewall.
+    services.lnd.restAddress = mkIf lndconnect "0.0.0.0";
+    # clightning-rest always listens on "0.0.0.0"
+
+    nix-bitcoin.secrets = {
+      wg-server-private-key = {};
+      wg-server-public-key = { user = wgConnectUser; group = "root"; };
+      wg-peer-private-key = { user = wgConnectUser; group = "root"; };
+      wg-peer-public-key = {};
+    };
+
+    nix-bitcoin.generateSecretsCmds.wireguard = let
+      wg = "${pkgs.wireguard-tools}/bin/wg";
+    in ''
+      makeWireguardKey() {
+        local name=$1
+        local priv=wg-$name-private-key
+        local pub=wg-$name-public-key
+        if [[ ! -e $priv ]]; then
+          ${wg} genkey > $priv
+        fi
+        if [[ $priv -nt $pub ]]; then
+          ${wg} pubkey < $priv > $pub
+        fi
+      }
+      makeWireguardKey server
+      makeWireguardKey peer
+    '';
+  };
+}

modules/rtl.nix

@@ -7,33 +7,33 @@ let
     address = mkOption {
       type = types.str;
       default = "127.0.0.1";
-      description = "HTTP server address.";
+      description = mdDoc "HTTP server address.";
     };
     port = mkOption {
       type = types.port;
       default = 3000;
-      description = "HTTP server port.";
+      description = mdDoc "HTTP server port.";
     };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/rtl";
-      description = "The data directory for RTL.";
+      description = mdDoc "The data directory for RTL.";
     };
     nodes = {
       clightning = {
         enable = mkOption {
           type = types.bool;
           default = false;
-          description = "Enable the clightning node interface.";
+          description = mdDoc "Enable the clightning node interface.";
         };
         extraConfig = mkOption {
-          type = types.attrs;
+          type = with types; attrsOf anything;
           default = {};
           example = {
             Settings.userPersona = "MERCHANT";
             Settings.logLevel = "DEBUG";
           };
-          description = ''
+          description = mdDoc ''
             Extra clightning node configuration.
             See here for all available options:
             https://github.com/Ride-The-Lightning/RTL/blob/master/.github/docs/Application_configurations.md
@@ -44,21 +44,21 @@ let
         enable = mkOption {
           type = types.bool;
           default = false;
-          description = "Enable the lnd node interface.";
+          description = mdDoc "Enable the lnd node interface.";
         };
         loop = mkOption {
           type = types.bool;
           default = false;
-          description = "Enable swaps with lightning-loop.";
+          description = mdDoc "Enable swaps with lightning-loop.";
         };
         extraConfig = mkOption {
-          type = types.attrs;
+          type = with types; attrsOf anything;
           default = {};
           example = {
             Settings.userPersona = "MERCHANT";
             Settings.logLevel = "DEBUG";
           };
-          description = ''
+          description = mdDoc ''
             Extra lnd node configuration.
             See here for all available options:
             https://github.com/Ride-The-Lightning/RTL/blob/master/.github/docs/Application_configurations.md
@@ -68,7 +68,7 @@ let
       reverseOrder = mkOption {
         type = types.bool;
         default = false;
-        description = ''
+        description = mdDoc ''
           Reverse the order of nodes shown in the UI.
           By default, clightning is shown before lnd.
         '';
@@ -77,28 +77,28 @@ let
     nightTheme = mkOption {
       type = types.bool;
       default = false;
-      description = "Enable the Night UI Theme.";
+      description = mdDoc "Enable the Night UI Theme.";
     };
     extraCurrency = mkOption {
       type = with types; nullOr str;
       default = null;
       example = "USD";
-      description = ''
+      description = mdDoc ''
         Currency code (ISO 4217) of the extra currency used for displaying balances.
         When set, this option enables online currency rate fetching.
         Warning: Rate fetching requires outgoing clearnet connections, so option
-        `tor.enforce` is automatically disabled.
+        {option}`tor.enforce` is automatically disabled.
       '';
     };
     user = mkOption {
       type = types.str;
       default = "rtl";
-      description = "The user as which to run RTL.";
+      description = mdDoc "The user as which to run RTL.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run RTL.";
+      description = mdDoc "The group as which to run RTL.";
     };
     tor.enforce = nbLib.tor.enforce;
   };
@@ -191,6 +191,7 @@ in {
                  optional cfg.nodes.lnd.enable "lnd.service";
       after = requires;
       environment.RTL_CONFIG_PATH = cfg.dataDir;
+      environment.DB_DIRECTORY_PATH = cfg.dataDir;
       serviceConfig = nbLib.defaultHardening // {
         ExecStartPre = [
           (nbLib.script "rtl-setup-config" ''

modules/secrets/secrets.nix

@@ -6,14 +6,14 @@ let
     secretsDir = mkOption {
       type = types.path;
       default = "/etc/nix-bitcoin-secrets";
-      description = "Directory to store secrets";
+      description = mdDoc "Directory to store secrets";
     };
 
     setupSecrets = mkOption {
       type = types.bool;
       default = false;
-      description = ''
-        Set permissions for existing secrets in `nix-bitcoin.secretsDir`
+      description = mdDoc ''
+        Set permissions for existing secrets in {option}`nix-bitcoin.secretsDir`
         before services are started.
       '';
     };
@@ -21,16 +21,16 @@ let
     generateSecrets = mkOption {
       type = types.bool;
       default = false;
-      description = ''
+      description = mdDoc ''
         Automatically generate all required secrets before services are started.
         Note: Make sure to create a backup of the generated secrets.
       '';
     };
 
     generateSecretsCmds = mkOption {
-      type = types.attrsOf types.str;
+      type = types.attrsOf types.lines;
       default = {};
-      description = ''
+      description = mdDoc ''
         Bash expressions for generating secrets.
       '';
     };
@@ -38,7 +38,7 @@ let
     # Currently, this is used only by ../deployment/nixops.nix
     deployment.secretsDir = mkOption {
       type = types.path;
-      description = ''
+      description = mdDoc ''
         Directory of local secrets that are transferred to the nix-bitcoin node on deployment
       '';
     };

modules/security.nix

@@ -6,16 +6,16 @@ with lib;
     nix-bitcoin.security.dbusHideProcessInformation = mkOption {
       type = types.bool;
       default = false;
-      description = ''
-        Only allow users with group 'proc' to retrieve systemd unit information like
+      description = mdDoc ''
+        Only allow users with group `proc` to retrieve systemd unit information like
         cgroup paths (i.e. (sub)process command lines) via D-Bus.
 
         This mitigates a systemd security issue where (sub)process command lines can
         be retrieved by services even when their access to /proc is restricted
         (via ProtectProc).
 
-        This option works by restricting the D-Bus method 'GetUnitProcesses', which
-        is also used internally by `systemctl status`.
+        This option works by restricting the D-Bus method `GetUnitProcesses`, which
+        is also used internally by {command}`systemctl status`.
       '';
     };
   };

modules/spark-wallet.nix

@@ -7,22 +7,22 @@ let
     address = mkOption {
       type = types.str;
       default = "localhost";
-      description = "http(s) server address.";
+      description = mdDoc "http(s) server address.";
     };
     port = mkOption {
       type = types.port;
       default = 9737;
-      description = "http(s) server port.";
+      description = mdDoc "http(s) server port.";
     };
     extraArgs = mkOption {
       type = types.separatedString " ";
       default = "";
-      description = "Extra command line arguments passed to spark-wallet.";
+      description = mdDoc "Extra command line arguments passed to spark-wallet.";
     };
     getPublicAddressCmd = mkOption {
       type = types.str;
       default = "";
-      description = ''
+      description = mdDoc ''
         Bash expression which outputs the public service address.
         If set, spark-wallet prints a QR code to the systemd journal which
         encodes an URL for accessing the web interface.
@@ -31,12 +31,12 @@ let
     user = mkOption {
       type = types.str;
       default = "spark-wallet";
-      description = "The user as which to run spark-wallet.";
+      description = mdDoc "The user as which to run spark-wallet.";
     };
     group = mkOption {
       type = types.str;
       default = cfg.user;
-      description = "The group as which to run spark-wallet.";
+      description = mdDoc "The group as which to run spark-wallet.";
     };
     tor = nbLib.tor;
   };

modules/versioning.nix

@@ -11,7 +11,7 @@ let
     nix-bitcoin.configVersion = mkOption {
       type = with types; nullOr str;
       default = null;
-      description = ''
+      description = mdDoc ''
         Set this option to the nix-bitcoin release version that your config is
         compatible with.
 
@@ -228,7 +228,16 @@ let
       version = "0.0.70";
       condition = config.services.lnd.lndconnectOnion.enable;
       message = ''
-        The `lndconnect-rest-onion` binary has been renamed to `lndconnect-onion`.
+        The `lndconnect-rest-onion` binary has been renamed to `lndconnect`.
+      '';
+    }
+    {
+      version = "0.0.85";
+      condition = config.services.fulcrum.enable;
+      message = ''
+        Fulcrum 1.9.0 has changed its database format.
+        The database update happens automatically and instantly on deployment,
+        but you can't switch back to an older Fulcrum version afterwards.
       '';
     }
   ];

pkgs/clightning-plugins/default.nix

@@ -6,8 +6,8 @@ let
   src = pkgs.fetchFromGitHub {
     owner = "lightningd";
     repo = "plugins";
-    rev = "59bad754cb338872c622ad716e8ed0063edf4052";
-    sha256 = "19nnmv2jvb0xmcha79dij399avasn2x521n9qg11lqj8xnzadm6a";
+    rev = "e625369423b00c70b23641662f62ccd898286edc";
+    sha256 = "04f30xlfr7pgdmdgka87x7sc9j82wc4zv7fbiqrjsc83dkmly81i";
   };
 
   version = builtins.substring 0 7 src.rev;
@@ -31,8 +31,8 @@ let
       description = "Lightning node exporter for the prometheus timeseries server";
       extraPkgs = [ prometheus_client ];
       patchRequirements =
-        "--replace prometheus-client==0.6.0 prometheus-client==0.13.1"
-        + " --replace pyln-client~=0.9.3 pyln-client~=0.11.1";
+        "--replace prometheus-client==0.6.0 prometheus-client==0.15.0"
+        + " --replace pyln-client~=0.9.3 pyln-client~=23.02";
     };
     rebalance = {
       description = "Keeps your channels balanced";
@@ -80,7 +80,7 @@ let
         inherit (plugin) description;
         homepage = "https://github.com/lightningd/plugins";
         license = licenses.bsd3;
-        maintainers = with maintainers; [ nixbitcoin earvstedt ];
+        maintainers = with maintainers; [ nixbitcoin erikarvstedt ];
         platforms = platforms.unix;
       };
     };

pkgs/clightning-rest/default.nix

@@ -49,7 +49,7 @@ let self = stdenvNoCC.mkDerivation {
     description = "REST API for C-Lightning";
     homepage = "https://github.com/Ride-The-Lightning/c-lightning-REST";
     license = licenses.mit;
-    maintainers = with maintainers; [ nixbitcoin earvstedt ];
+    maintainers = with maintainers; [ nixbitcoin erikarvstedt ];
     platforms = platforms.unix;
   };
 }; in self

pkgs/default.nix

@@ -13,17 +13,24 @@ let self = {
   clightning-rest = pkgs.callPackage ./clightning-rest { inherit (self) fetchNodeModules; };
   clboss = pkgs.callPackage ./clboss { };
   clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages);
-  joinmarket = pkgs.callPackage ./joinmarket { nbPythonPackageOverrides = import ./python-packages self; };
+  joinmarket = pkgs.callPackage ./joinmarket { inherit (self) nbPython3PackagesJoinmarket; };
   lndinit = pkgs.callPackage ./lndinit { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
   rtl = pkgs.callPackage ./rtl { inherit (self) fetchNodeModules; };
   # The secp256k1 version used by joinmarket
   secp256k1 = pkgs.callPackage ./secp256k1 { };
   spark-wallet = pkgs.callPackage ./spark-wallet { };
+  trustedcoin = pkgs.callPackage ./trustedcoin { };
 
-  nbPython3Packages = (pkgs.python3.override {
-    packageOverrides = import ./python-packages self;
-  }).pkgs;
+  # TODO-EXTERNAL:
+  # Remove this when https://github.com/lightningnetwork/lnd/pull/7672
+  # has been resolved
+  lnd = pkgsUnstable.callPackage ./lnd { };
+
+  pyPkgs = import ./python-packages self pkgs.python3;
+  inherit (self.pyPkgs)
+    nbPython3Packages
+    nbPython3PackagesJoinmarket;
 
   fetchNodeModules = pkgs.callPackage ./build-support/fetch-node-modules.nix { };
 

pkgs/joinmarket/default.nix

@@ -1,23 +1,15 @@
-{ stdenv, lib, fetchurl, python3, nbPythonPackageOverrides, pkgs }:
+{ stdenv, lib, fetchFromGitHub, python3, nbPython3PackagesJoinmarket }:
 
 let
-  version = "0.9.7";
-  src = fetchurl {
-    url = "https://github.com/JoinMarket-Org/joinmarket-clientserver/archive/v${version}.tar.gz";
-    sha256 = "13bfr8ha6bka8wiai8m79ki43dn2r311lrfffr39ni2wy1v12l93";
+  version = "0.9.9";
+  src = fetchFromGitHub {
+    owner = "joinmarket-org";
+    repo = "joinmarket-clientserver";
+    rev = "v${version}";
+    sha256 = "sha256-dkeSgAhjNl8o/ATKYAlQxxCrur5fLdXuMDXSnWaxYP8=";
   };
 
-  pyPkgs = (python3.override {
-    packageOverrides = (self: super: let
-      overrides = nbPythonPackageOverrides self super;
-    in
-      overrides // {
-        cryptography = overrides.cryptography_3_3_2;
-      }
-    );
-  }).pkgs;
-
-  runtimePackages = with pyPkgs; [
+  runtimePackages = with nbPython3PackagesJoinmarket; [
     joinmarketbase
     joinmarketclient
     joinmarketbitcoin

pkgs/joinmarket/get-sha256.sh

@@ -1,25 +1,23 @@
-#!/usr/bin/env bash
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p git gnupg jq
+
 set -euo pipefail
-. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "git gnupg" "$@"
+newVersion=$(curl -s "https://api.github.com/repos/joinmarket-org/joinmarket-clientserver/releases" | jq -r '.[0].tag_name')
 
-TMPDIR="$(mktemp -d -p /tmp)"
-trap 'rm -rf $TMPDIR' EXIT
-cd "$TMPDIR"
-
-echo "Fetching latest release"
-git clone https://github.com/joinmarket-org/joinmarket-clientserver 2> /dev/null
-cd joinmarket-clientserver
-latest=$(git describe --tags "$(git rev-list --tags --max-count=1)")
-echo "Latest release is $latest"
-
-# GPG verification
-export GNUPGHOME=$TMPDIR
+# Fetch release and GPG-verify the content hash
+tmpdir=$(mktemp -d /tmp/joinmarket-verify-gpg.XXX)
+repo=$tmpdir/repo
+git clone --depth 1 --branch "${newVersion}" -c advice.detachedHead=false https://github.com/joinmarket-org/joinmarket-clientserver "$repo"
+export GNUPGHOME=$tmpdir
 echo "Fetching Adam Gibson's key"
 gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 2B6FC204D9BF332D062B461A141001A1AF77F20B 2> /dev/null
-echo "Verifying latest release"
-git verify-tag "$latest"
+echo
+echo "Verifying commit"
+git -C "$repo" verify-commit HEAD
+rm -rf "$repo"/.git
+newHash=$(nix hash path "$repo")
+rm -rf "$tmpdir"
+echo
 
-echo "tag: $latest"
-# The prefix option is necessary because GitHub prefixes the archive contents in this format
-echo "sha256: $(nix-hash --type sha256 --flat --base32 \
-                <(git archive --format tar.gz --prefix=joinmarket-clientserver-"${latest//v}"/ "$latest"))"
+echo "tag: $newVersion"
+echo "hash: $newHash"

pkgs/lib.nix

@@ -1,4 +1,4 @@
-lib: pkgs:
+lib: pkgs: config:
 
 with lib;
 
@@ -7,33 +7,33 @@ with lib;
 let self = {
   # These settings roughly follow systemd's "strict" security profile
   defaultHardening = {
-      PrivateTmp = "true";
+      PrivateTmp = true;
       ProtectSystem = "strict";
-      ProtectHome = "true";
-      NoNewPrivileges = "true";
-      PrivateDevices = "true";
-      MemoryDenyWriteExecute = "true";
-      ProtectKernelTunables = "true";
-      ProtectKernelModules = "true";
-      ProtectKernelLogs = "true";
-      ProtectClock = "true";
+      ProtectHome = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      MemoryDenyWriteExecute = true;
+      ProtectKernelTunables = true;
+      ProtectKernelModules = true;
+      ProtectKernelLogs = true;
+      ProtectClock = true;
       ProtectProc = "invisible";
       ProcSubset = "pid";
-      ProtectControlGroups = "true";
+      ProtectControlGroups = true;
       RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
-      RestrictNamespaces = "true";
-      LockPersonality = "true";
+      RestrictNamespaces = true;
+      LockPersonality = true;
       IPAddressDeny = "any";
-      PrivateUsers = "true";
-      RestrictSUIDSGID = "true";
-      RemoveIPC = "true";
-      RestrictRealtime = "true";
-      ProtectHostname = "true";
+      PrivateUsers = true;
+      RestrictSUIDSGID = true;
+      RemoveIPC = true;
+      RestrictRealtime = true;
+      ProtectHostname = true;
       CapabilityBoundingSet = "";
       # @system-service whitelist and docker seccomp blacklist (except for "clone"
       # which is a core requirement for systemd services)
       # @system-service is defined in src/shared/seccomp-util.c (systemd source)
-      SystemCallFilter = [ "@system-service" "~add_key kcmp keyctl mbind move_pages name_to_handle_at personality process_vm_readv process_vm_writev request_key set_mempolicy setns unshare userfaultfd" ];
+      SystemCallFilter = [ "@system-service" "~add_key kcmp keyctl mbind move_pages name_to_handle_at personality process_vm_readv process_vm_writev request_key setns unshare userfaultfd" ];
       SystemCallArchitectures = "native";
   };
 
@@ -42,7 +42,7 @@ let self = {
   };
 
   # nodejs applications require memory write execute for JIT compilation
-  nodejs = { MemoryDenyWriteExecute = "false"; };
+  nodejs = { MemoryDenyWriteExecute = false; };
 
   # Allow takes precedence over Deny.
   allowLocalIPAddresses = {
@@ -115,4 +115,8 @@ let self = {
       (map (ip: "IP:${ip}") cert.extraIPs)
     );
 
+  test = {
+    mkIfTest = test: mkIf (config.tests.${test} or false);
+  };
+
 }; in self

pkgs/lnd/default.nix

@@ -0,0 +1,12 @@
+{ lnd, fetchpatch }:
+
+lnd.overrideAttrs (_: {
+  patches = [
+    (fetchpatch {
+      # https://github.com/lightningnetwork/lnd/pull/7672
+      name = "fix-PKCS8-cert-key-support";
+      url = "https://github.com/lightningnetwork/lnd/commit/bfdd5db0d97a6d65489d980a917bbd2243dfe15c.patch";
+      hash = "sha256-j9EirxyNi48DGzLuHcZ36LrFlbJLXrE8L+1TYh5Yznk=";
+    })
+  ];
+})

pkgs/lndinit/default.nix

@@ -19,6 +19,6 @@ buildGoModule rec {
     description = "Wallet initializer utility for lnd";
     homepage = "https://github.com/lightninglabs/lndinit";
     license = licenses.mit;
-    maintainers = with maintainers; [ earvstedt ];
+    maintainers = with maintainers; [ erikarvstedt ];
   };
 }

pkgs/nixpkgs-pinned.nix

@@ -16,5 +16,5 @@ let
 in
 {
   nixpkgs = fetch lockedInputs.nixpkgs;
-  nixpkgs-unstable = fetch lockedInputs.nixpkgsUnstable;
+  nixpkgs-unstable = fetch lockedInputs.nixpkgs-unstable;
 }

pkgs/pinned.nix

@@ -4,21 +4,20 @@ pkgs: pkgsUnstable:
   inherit (pkgs)
     bitcoin
     bitcoind
-    electrs
-    elementsd
     extra-container
     lightning-pool
-    lndconnect
-    nbxplorer;
+    lndconnect;
 
   inherit (pkgsUnstable)
     btcpayserver
     charge-lnd
     clightning
+    electrs
+    elementsd
     fulcrum
     hwi
     lightning-loop
-    lnd;
+    nbxplorer;
 
   inherit pkgs pkgsUnstable;
 }

pkgs/python-packages/bencoderpyx/default.nix

@@ -2,11 +2,11 @@
 
 buildPythonPackage rec {
   pname = "bencoder.pyx";
-  version = "2.0.1";
+  version = "3.0.1";
 
   src = fetchurl {
-    url = "https://github.com/whtsky/bencoder.pyx/archive/v${version}.tar.gz";
-    sha256 = "f3ff92ac706a7e4692bed5e6cbe205963327f3076f55e408eb948659923eac72";
+    url = "https://github.com/whtsky/bencoder.pyx/archive/9a47768f3ceba9df9e6fbaa7c445f59960889009.tar.gz";
+    sha256 = "1yh565xjbbhn49xjfms80ac8psjbzn66n8dcx0x8mn7zzjv06clz";
   };
 
   nativeBuildInputs = [ cython ];

pkgs/python-packages/default.nix

@@ -1,51 +1,63 @@
-nbPkgs: self: super:
-let
+nbPkgs: python3:
+rec {
+  pyPkgsOverrides = self: super: let
     inherit (self) callPackage;
-
-  joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; };
     clightningPkg = pkg: callPackage pkg { inherit (nbPkgs.pinned) clightning; };
-
-  unstable = (import ../nixpkgs-pinned.nix).nixpkgs-unstable;
-in {
-  bencoderpyx = callPackage ./bencoderpyx {};
-  chromalog = callPackage ./chromalog {};
+  in
+    {
       coincurve = callPackage ./coincurve {};
-  python-bitcointx = callPackage ./python-bitcointx { inherit (nbPkgs) secp256k1; };
-  runes = callPackage ./runes {};
-  sha256 = callPackage ./sha256 {};
       txzmq = callPackage ./txzmq {};
-  urldecode = callPackage ./urldecode {};
-
-  joinmarketbase = joinmarketPkg ./jmbase;
-  joinmarketclient = joinmarketPkg ./jmclient;
-  joinmarketbitcoin = joinmarketPkg ./jmbitcoin;
-  joinmarketdaemon = joinmarketPkg ./jmdaemon;
 
       pyln-client = clightningPkg ./pyln-client;
       pyln-proto = clightningPkg ./pyln-proto;
       pyln-bolt7 = clightningPkg ./pyln-bolt7;
       pylightning = clightningPkg ./pylightning;
 
-  # Don't mark `klein` as broken.
-  # `klein` is fixed by using werkzeug 2.1.0 (see below)
-  klein = super.klein.overrideAttrs (old: {
-    meta = builtins.removeAttrs old.meta [ "broken" ];
-  });
+      # Packages only used by joinmarket
+      bencoderpyx = callPackage ./bencoderpyx {};
+      chromalog = callPackage ./chromalog {};
+      python-bitcointx = callPackage ./python-bitcointx {
+        inherit (nbPkgs) secp256k1;
+        openssl = super.pkgs.openssl_1_1;
+      };
+      runes = callPackage ./runes {};
+      sha256 = callPackage ./sha256 {};
+    };
+
+  # Joinmarket requires a custom package set because it uses older versions of Python pkgs
+  pyPkgsOverridesJoinmarket = self: super: let
+    inherit (self) callPackage;
+    joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; };
+  in
+    (pyPkgsOverrides self super) // {
+      joinmarketbase = joinmarketPkg ./jmbase;
+      joinmarketclient = joinmarketPkg ./jmclient;
+      joinmarketbitcoin = joinmarketPkg ./jmbitcoin;
+      joinmarketdaemon = joinmarketPkg ./jmdaemon;
 
       ## Specific versions of packages that already exist in nixpkgs
 
       # cryptography 3.3.2, required by joinmarketdaemon
-  # Used in the private python package set for joinmarket (../joinmarket/default.nix)
-  cryptography_3_3_2 = callPackage ./specific-versions/cryptography {
+      cryptography = callPackage ./specific-versions/cryptography {
+        openssl = super.pkgs.openssl_1_1;
         cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {};
       };
 
       # autobahn 20.12.3, required by joinmarketclient
       autobahn = callPackage ./specific-versions/autobahn.nix {};
 
-  # werkzeug 2.1.0, required by jmclient (via pkg `klein`)
-  werkzeug = callPackage ./specific-versions/werkzeug.nix {};
-
-  # pyopenssl 20.0.1, required by joinmarketdaemon
+      # pyopenssl 21.0.0, required by joinmarketdaemon
       pyopenssl = callPackage ./specific-versions/pyopenssl.nix {};
+
+      # twisted 22.4.0, required by joinmarketbase
+      twisted = callPackage ./specific-versions/twisted.nix {};
+    };
+
+  nbPython3Packages = (python3.override {
+    packageOverrides = pyPkgsOverrides;
+  }).pkgs;
+
+  nbPython3PackagesJoinmarket = (python3.override {
+    packageOverrides = pyPkgsOverridesJoinmarket;
+  }).pkgs;
 }

pkgs/python-packages/jmbitcoin/default.nix

@@ -1,4 +1,4 @@
-{ version, src, lib, buildPythonPackage, fetchurl, urldecode, pyaes, python-bitcointx, joinmarketbase }:
+{ version, src, lib, buildPythonPackage, fetchurl, pyaes, python-bitcointx, joinmarketbase }:
 
 buildPythonPackage rec {
   pname = "joinmarketbitcoin";
@@ -6,7 +6,7 @@ buildPythonPackage rec {
 
   postUnpack = "sourceRoot=$sourceRoot/jmbitcoin";
 
-  propagatedBuildInputs = [ urldecode pyaes python-bitcointx ];
+  propagatedBuildInputs = [ pyaes python-bitcointx ];
 
   checkInputs = [ joinmarketbase ];
 

pkgs/python-packages/jmclient/default.nix

@@ -12,9 +12,9 @@ buildPythonPackage rec {
 
   patchPhase = ''
     substituteInPlace setup.py \
-      --replace "'klein==20.6.0'" "'klein==21.8.0'"
+      --replace "'klein==20.6.0'" "'klein>=20.6.0'"
     substituteInPlace setup.py \
-      --replace "'pyjwt==2.1.0'" "'pyjwt==2.4.0'"
+      --replace "'pyjwt==2.4.0'" "'pyjwt==2.5.0'"
   '';
 
   meta = with lib; {

pkgs/python-packages/jmdaemon/default.nix

@@ -8,6 +8,12 @@ buildPythonPackage rec {
 
   propagatedBuildInputs = [ txtorcon cryptography pyopenssl libnacl joinmarketbase ];
 
+  # libnacl 1.8.0 is not on github
+  patchPhase = ''
+    substituteInPlace setup.py \
+      --replace "'libnacl==1.8.0'" "'libnacl==1.7.2'"
+  '';
+
   meta = with lib; {
     description = "Client library for Bitcoin coinjoins";
     homepage = "https://github.com/Joinmarket-Org/joinmarket-clientserver";

pkgs/python-packages/pyln-client/default.nix

@@ -16,13 +16,5 @@ buildPythonPackage rec {
 
   checkInputs = [ pytestCheckHook ];
 
-  # TODO-EXTERNAL:
-  # This patch is a variant (fixed relative path) of
-  # https://github.com/ElementsProject/lightning/pull/5574. This is already
-  # fixed upstream. Remove this after the next clightning release.
-  patches = [
-    ./msat-null.patch
-  ];
-
   postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
 }

pkgs/python-packages/pyln-client/msat-null.patch

@@ -1,16 +0,0 @@
-diff --git a/pyln/client/lightning.py b/pyln/client/lightning.py
-index 38fc7563f..0013b89a3 100644
---- a/pyln/client/lightning.py
-+++ b/pyln/client/lightning.py
-@@ -455,6 +455,11 @@ class LightningRpc(UnixDomainSocketRpc):
-                     if k.endswith('msat'):
-                         if isinstance(v, list):
-                             obj[k] = [Millisatoshi(e) for e in v]
-+                        # FIXME: Deprecated "listconfigs" gives two 'null' fields:
-+                        #            "lease-fee-base-msat": null,
-+                        #            "channel-fee-max-base-msat": null,
-+                        elif v is None:
-+                            obj[k] = None
-                         else:
-                             obj[k] = Millisatoshi(v)
-                     else:

pkgs/python-packages/pyln-proto/default.nix

@@ -27,4 +27,8 @@ buildPythonPackage rec {
   checkInputs = [ pytestCheckHook ];
 
   postUnpack = "sourceRoot=$sourceRoot/contrib/pyln-proto";
+
+  postPatch = ''
+    sed -i 's|cryptography = "^36.0.1"|cryptography = "^38.0.0"|' pyproject.toml
+  '';
 }

pkgs/python-packages/specific-versions/pyopenssl.nix

@@ -6,17 +6,50 @@
 , cryptography
 , pyasn1
 , idna
-, pytest
+, pytestCheckHook
 , pretend
 , flaky
 , glibcLocales
 , six
 }:
 
-let
+buildPythonPackage rec {
+  pname = "pyopenssl";
+  version = "21.0.0";
+
+  src = fetchPypi {
+    pname = "pyOpenSSL";
+    inherit version;
+    sha256 = "5e2d8c5e46d0d865ae933bef5230090bdaf5506281e9eec60fa250ee80600cb3";
+  };
+
+  outputs = [ "out" "dev" ];
+
+  # Seems to fail unpredictably on Darwin. See https://hydra.nixos.org/build/49877419/nixlog/1
+  # for one example, but I've also seen ContextTests.test_set_verify_callback_exception fail.
+  doCheck = !stdenv.isDarwin;
+
+  nativeBuildInputs = [ openssl ];
+  propagatedBuildInputs = [ cryptography pyasn1 idna six ];
+
+  checkInputs = [ pytestCheckHook pretend flaky glibcLocales ];
+
+  preCheck = ''
+    export LANG="en_US.UTF-8"
+  '';
+
+  disabledTests = [
+    # https://github.com/pyca/pyopenssl/issues/692
+    # These tests, we disable always.
+    "test_set_default_verify_paths"
+    "test_fallback_default_verify_paths"
+    # https://github.com/pyca/pyopenssl/issues/768
+    "test_wantWriteError"
+    # https://github.com/pyca/pyopenssl/issues/1043
+    "test_alpn_call_failure"
+  ] ++ lib.optionals (lib.hasPrefix "libressl" openssl.meta.name) [
     # https://github.com/pyca/pyopenssl/issues/791
     # These tests, we disable in the case that libressl is passed in as openssl.
-  failingLibresslTests = [
     "test_op_no_compression"
     "test_npn_advertise_error"
     "test_npn_select_error"
@@ -29,64 +62,21 @@ let
     "test_verify_with_revoked"
     "test_set_notAfter"
     "test_set_notBefore"
-  ];
-
-  # these tests are extremely tightly wed to the exact output of the openssl cli tool,
-  # including exact punctuation.
-  failingOpenSSL_1_1Tests = [
+  ] ++ lib.optionals (lib.versionAtLeast (lib.getVersion openssl.name) "1.1") [
+    # these tests are extremely tightly wed to the exact output of the openssl cli tool, including exact punctuation.
     "test_dump_certificate"
     "test_dump_privatekey_text"
     "test_dump_certificate_request"
     "test_export_text"
+  ] ++ lib.optionals stdenv.is32bit [
+    # https://github.com/pyca/pyopenssl/issues/974
+    "test_verify_with_time"
   ];
 
-  disabledTests = [
-    # https://github.com/pyca/pyopenssl/issues/692
-    # These tests, we disable always.
-    "test_set_default_verify_paths"
-    "test_fallback_default_verify_paths"
-    # https://github.com/pyca/pyopenssl/issues/768
-    "test_wantWriteError"
-  ] ++ (
-    lib.optionals (lib.hasPrefix "libressl" openssl.meta.name) failingLibresslTests
-  ) ++ (
-    lib.optionals (lib.versionAtLeast (lib.getVersion openssl.name) "1.1") failingOpenSSL_1_1Tests
-  ) ++ (
-    # https://github.com/pyca/pyopenssl/issues/974
-    lib.optionals stdenv.is32bit [ "test_verify_with_time" ]
-  );
-
-  # Compose the final string expression, including the "-k" and the single quotes.
-  testExpression = lib.optionalString (disabledTests != [])
-    "-k 'not ${lib.concatStringsSep " and not " disabledTests}'";
-
-in
-
-buildPythonPackage rec {
-  pname = "pyopenssl";
-  version = "20.0.1";
-
-  src = fetchPypi {
-    pname = "pyOpenSSL";
-    inherit version;
-    sha256 = "4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51";
+  meta = with lib; {
+    description = "Python wrapper around the OpenSSL library";
+    homepage = "https://github.com/pyca/pyopenssl";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ SuperSandro2000 ];
   };
-
-  outputs = [ "out" "dev" ];
-
-  checkPhase = ''
-    runHook preCheck
-    export LANG="en_US.UTF-8"
-    py.test tests ${testExpression}
-    runHook postCheck
-  '';
-
-  # Seems to fail unpredictably on Darwin. See https://hydra.nixos.org/build/49877419/nixlog/1
-  # for one example, but I've also seen ContextTests.test_set_verify_callback_exception fail.
-  doCheck = !stdenv.isDarwin;
-
-  nativeBuildInputs = [ openssl ];
-  propagatedBuildInputs = [ cryptography pyasn1 idna six ];
-
-  checkInputs = [ pytest pretend flaky glibcLocales ];
 }

pkgs/python-packages/specific-versions/twisted.nix

@@ -0,0 +1,173 @@
+{ lib
+, stdenv
+, buildPythonPackage
+, pythonOlder
+, fetchPypi
+, python
+, appdirs
+, attrs
+, automat
+, bcrypt
+, constantly
+, contextvars
+, cryptography
+, git
+, glibcLocales
+, h2
+, hyperlink
+, idna
+, incremental
+, priority
+, pyasn1
+, pyhamcrest
+, pynacl
+, pyopenssl
+, pyserial
+, service-identity
+, setuptools
+, typing-extensions
+, zope_interface
+
+  # for passthru.tests
+, cassandra-driver
+, klein
+, magic-wormhole
+, scrapy
+, treq
+, txaio
+, txamqp
+, txrequests
+, txtorcon
+, thrift
+, nixosTests
+}:
+
+buildPythonPackage rec {
+  pname = "twisted";
+  version = "22.4.0";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.6";
+
+  src = fetchPypi {
+    pname = "Twisted";
+    inherit version;
+    extension = "tar.gz";
+    sha256 = "sha256-oEeZD1ffrh4L0rffJSbU8W3NyEN3TcEIt4xS8qXxNoA=";
+  };
+
+  __darwinAllowLocalNetworking = true;
+
+  propagatedBuildInputs = [
+    attrs
+    automat
+    constantly
+    hyperlink
+    incremental
+    setuptools
+    typing-extensions
+    zope_interface
+  ];
+
+  postPatch = ''
+    echo 'ListingTests.test_localeIndependent.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
+    echo 'ListingTests.test_newFile.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
+    echo 'ListingTests.test_newSingleDigitDayOfMonth.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
+    echo 'ListingTests.test_oldFile.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
+    echo 'ListingTests.test_oldSingleDigitDayOfMonth.skip = "Timezone issue"'>> src/twisted/conch/test/test_cftp.py
+
+    echo 'PTYProcessTestsBuilder_AsyncioSelectorReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
+    echo 'PTYProcessTestsBuilder_SelectReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
+
+    echo 'UNIXTestsBuilder_AsyncioSelectorReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
+    echo 'UNIXTestsBuilder_SelectReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
+
+    echo 'FileObserverTests.test_getTimezoneOffsetEastOfUTC.skip = "mktime argument out of range"'>> src/twisted/test/test_log.py
+    echo 'FileObserverTests.test_getTimezoneOffsetWestOfUTC.skip = "mktime argument out of range"'>> src/twisted/test/test_log.py
+    echo 'FileObserverTests.test_getTimezoneOffsetWithoutDaylightSavingTime.skip = "tuple differs, values not"'>> src/twisted/test/test_log.py
+
+    echo 'MulticastTests.test_joinLeave.skip = "No such device"'>> src/twisted/test/test_udp.py
+    echo 'MulticastTests.test_loopback.skip = "No such device"'>> src/twisted/test/test_udp.py
+    echo 'MulticastTests.test_multicast.skip = "Reactor was unclean"'>> src/twisted/test/test_udp.py
+    echo 'MulticastTests.test_multiListen.skip = "No such device"'>> src/twisted/test/test_udp.py
+
+    echo 'DomishExpatStreamTests.test_namespaceWithWhitespace.skip = "syntax error: line 1, column 0"'>> src/twisted/words/test/test_domish.py
+
+    # not packaged
+    substituteInPlace src/twisted/test/test_failure.py \
+      --replace "from cython_test_exception_raiser import raiser  # type: ignore[import]" "raiser = None"
+  '' + lib.optionalString stdenv.isLinux ''
+    echo 'PTYProcessTestsBuilder_EPollReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
+    echo 'PTYProcessTestsBuilder_PollReactorTests.test_openFileDescriptors.skip = "invalid syntax"'>> src/twisted/internet/test/test_process.py
+    echo 'UNIXTestsBuilder_EPollReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
+    echo 'UNIXTestsBuilder_PollReactorTests.test_sendFileDescriptorTriggersPauseProducing.skip = "sendFileDescriptor producer was not paused"'>> src/twisted/internet/test/test_unix.py
+
+    # Patch t.p._inotify to point to libc. Without this,
+    # twisted.python.runtime.platform.supportsINotify() == False
+    substituteInPlace src/twisted/python/_inotify.py --replace \
+      "ctypes.util.find_library(\"c\")" "'${stdenv.cc.libc}/lib/libc.so.6'"
+  '' + lib.optionalString (stdenv.isAarch64 && stdenv.isDarwin) ''
+    echo 'AbortConnectionTests_AsyncioSelectorReactorTests.test_fullWriteBufferAfterByteExchange.skip = "Timeout after 120 seconds"' >> src/twisted/internet/test/test_tcp.py
+    echo 'AbortConnectionTests_AsyncioSelectorReactorTests.test_resumeProducingAbort.skip = "Timeout after 120 seconds"' >> src/twisted/internet/test/test_tcp.py
+  '';
+
+  # Generate Twisted's plug-in cache. Twisted users must do it as well. See
+  # http://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3
+  # and http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=477103 for details.
+  postFixup = ''
+    $out/bin/twistd --help > /dev/null
+  '';
+
+  checkInputs = [
+    git
+    glibcLocales
+    pyhamcrest
+  ]
+  ++ passthru.optional-dependencies.conch
+  # not supported on aarch64-darwin: https://github.com/pyca/pyopenssl/issues/873
+  ++ lib.optionals (!(stdenv.isDarwin && stdenv.isAarch64)) passthru.optional-dependencies.tls;
+
+  checkPhase = ''
+    export SOURCE_DATE_EPOCH=315532800
+    export PATH=$out/bin:$PATH
+    # race conditions when running in paralell
+    ${python.interpreter} -m twisted.trial twisted
+  '';
+
+  passthru = {
+    optional-dependencies = rec {
+      conch = [ appdirs bcrypt cryptography pyasn1 ];
+      conch_nacl = conch ++ [ pynacl ];
+      contextvars = lib.optionals (pythonOlder "3.7") [ contextvars ];
+      http2 = [ h2 priority ];
+      serial = [ pyserial ];
+      tls = [ idna pyopenssl service-identity ];
+    };
+
+    tests = {
+      inherit
+        cassandra-driver
+        klein
+        magic-wormhole
+        scrapy
+        treq
+        txaio
+        txamqp
+        txrequests
+        txtorcon
+        thrift;
+      inherit (nixosTests) buildbot matrix-synapse;
+    };
+  };
+
+  meta = with lib; {
+    homepage = "https://github.com/twisted/twisted";
+    description = "Twisted, an event-driven networking engine written in Python";
+    longDescription = ''
+      Twisted is an event-driven networking engine written in Python
+      and licensed under the MIT license.
+    '';
+    license = licenses.mit;
+    maintainers = with maintainers; [ SuperSandro2000 ];
+  };
+}

pkgs/python-packages/specific-versions/werkzeug.nix

@@ -1,68 +0,0 @@
-{ lib
-, stdenv
-, buildPythonPackage
-, pythonOlder
-, fetchPypi
-, watchdog
-, dataclasses
-, ephemeral-port-reserve
-, pytest-timeout
-, pytest-xprocess
-, pytestCheckHook
-}:
-
-buildPythonPackage rec {
-  pname = "werkzeug";
-  version = "2.1.0";
-  format = "setuptools";
-
-  disabled = pythonOlder "3.7";
-
-  src = fetchPypi {
-    pname = "Werkzeug";
-    inherit version;
-    sha256 = "sha256-m1VGaj6Z4TsfBoamYRfTm9qFqZIWbgp5rt/PNYYyj3o=";
-  };
-
-  propagatedBuildInputs = lib.optionals (!stdenv.isDarwin) [
-    # watchdog requires macos-sdk 10.13+
-    watchdog
-  ] ++ lib.optionals (pythonOlder "3.7") [
-    dataclasses
-  ];
-
-  checkInputs = [
-    ephemeral-port-reserve
-    pytest-timeout
-    pytest-xprocess
-    pytestCheckHook
-  ];
-
-  disabledTests = lib.optionals stdenv.isDarwin [
-    "test_get_machine_id"
-  ];
-
-  disabledTestPaths = [
-    # ConnectionRefusedError: [Errno 111] Connection refused
-    "tests/test_serving.py"
-  ];
-
-  pytestFlagsArray = [
-    # don't run tests that are marked with filterwarnings, they fail with
-    # warnings._OptionError: unknown warning category: 'pytest.PytestUnraisableExceptionWarning'
-    "-m 'not filterwarnings'"
-  ];
-
-  meta = with lib; {
-    homepage = "https://palletsprojects.com/p/werkzeug/";
-    description = "The comprehensive WSGI web application library";
-    longDescription = ''
-      Werkzeug is a comprehensive WSGI web application library. It
-      began as a simple collection of various utilities for WSGI
-      applications and has become one of the most advanced WSGI
-      utility libraries.
-    '';
-    license = licenses.bsd3;
-    maintainers = with maintainers; [ ];
-  };
-}

pkgs/python-packages/urldecode/default.nix

@@ -1,16 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi }:
-buildPythonPackage rec {
-  pname = "urldecode";
-  version = "0.1";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "0w8my7kdwxppsfzzi1b2cxhypm6r1fsrnb2hnd752axq4gfsddjj";
-  };
-
-  meta = with lib; {
-    description = "A simple function to decode an encoded url";
-    homepage = "https://github.com/jennyq/urldecode";
-    maintainers = with maintainers; [ nixbitcoin ];
-  };
-}

pkgs/rtl/default.nix

@@ -10,22 +10,11 @@
 }:
 let self = stdenvNoCC.mkDerivation {
   pname = "rtl";
-  version = "0.13.1";
+  version = "0.13.6";
 
-  src = applyPatches {
   src = fetchurl {
     url = "https://github.com/Ride-The-Lightning/RTL/archive/refs/tags/v${self.version}.tar.gz";
-      hash = "sha256-k40xwDDJxny1nPN2xz60WfbinxMNM0QPdglibO2anZw=";
-    };
-
-    patches = [
-      # Move non-runtime deps to `devDependencies`
-      # https://github.com/Ride-The-Lightning/RTL/pull/1070
-      (fetchpatch {
-        url = "https://github.com/Ride-The-Lightning/RTL/pull/1070.patch";
-        sha256 = "sha256-esDkYI27SNzj2AhYHS9XqlW0r2mr+o0K4A6PUE2kbWU=";
-      })
-    ];
+    hash = "sha256-eyRM28h2TV3IyW4hDPHj/wMJxLEZin7AqWQZGQt5mV4=";
   };
 
   passthru = {
@@ -34,7 +23,10 @@ let self = stdenvNoCC.mkDerivation {
 
     nodeModules = fetchNodeModules {
       inherit (self) src nodejs;
-      hash = "sha256-bYZ6snfXhDZ3MMga45EHVrPZxC0/Q0e3AgCgMBire64=";
+      # TODO-EXTERNAL: Remove `npmFlags` when no longer required
+      # See: https://github.com/Ride-The-Lightning/RTL/issues/1182
+      npmFlags = "--legacy-peer-deps";
+      hash = "sha256-C4yK6deYXPrTa383aXiHoO0w3JAMIfAaESCEy9KKY2k=";
     };
   };
 
@@ -68,7 +60,7 @@ let self = stdenvNoCC.mkDerivation {
     description = "A web interface for LND, c-lightning and Eclair";
     homepage = "https://github.com/Ride-The-Lightning/RTL";
     license = licenses.mit;
-    maintainers = with maintainers; [ nixbitcoin earvstedt ];
+    maintainers = with maintainers; [ nixbitcoin erikarvstedt ];
     platforms = platforms.unix;
   };
 }; in self

pkgs/rtl/generate.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 . "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "gnupg wget gnused" "$@"
 
-version="0.13.1"
+version="0.13.6"
 repo=https://github.com/Ride-The-Lightning/RTL
 
 scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)

pkgs/spark-wallet/composition.nix

@@ -5,7 +5,7 @@
   }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}:
 
 let
-  nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
+  nodeEnv = import (pkgs.path + "/pkgs/development/node-packages/node-env.nix") {
     inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
     inherit pkgs nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;

pkgs/spark-wallet/default.nix

@@ -10,7 +10,7 @@ nodePackages.package.override {
     description = "A minimalistic wallet GUI for c-lightning";
     homepage = "https://github.com/shesek/spark-wallet";
     license = licenses.mit;
-    maintainers = with maintainers; [ nixbitcoin earvstedt ];
+    maintainers = with maintainers; [ nixbitcoin erikarvstedt ];
     platforms = platforms.unix;
   };
 }

pkgs/trustedcoin/default.nix

@@ -0,0 +1,23 @@
+{ lib, buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "trustedcoin";
+  version = "0.6.1";
+  src = fetchFromGitHub {
+    owner = "nbd-wtf";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "sha256-UNQjxhAT0mK1In7vUtIoMoMNBV+0wkrwbDmm7m+0R3o=";
+  };
+
+  vendorSha256 = "sha256-xvkK9rMQlXTnNyOMd79qxVSvhgPobcBk9cq4/YWbupY=";
+
+  subPackages = [ "." ];
+
+  meta = with lib; {
+    description = "Light bitcoin node implementation";
+    homepage = "https://github.com/nbd-wtf/trustedcoin";
+    maintainers = with maintainers; [ seberm fort-nix ];
+    platforms = platforms.linux;
+  };
+}

pkgs/trustedcoin/get-sha256.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p git gnupg curl jq
+set -euo pipefail
+
+
+TMPDIR="$(mktemp -d -p /tmp)"
+trap 'rm -rf $TMPDIR' EXIT
+cd "$TMPDIR"
+
+echo "Fetching latest release"
+repo='nbd-wtf/trustedcoin'
+latest=$(curl --location --silent --show-error https://api.github.com/repos/${repo}/releases/latest | jq -r .tag_name)
+echo "Latest release is $latest"
+git clone --depth 1 --branch "$latest" "https://github.com/${repo}" 2>/dev/null
+cd trustedcoin
+
+echo "tag: $latest"
+git checkout -q "tags/$latest"
+rm -rf .git
+nix --extra-experimental-features nix-command hash path .

test/README.md

@@ -0,0 +1,84 @@
+The [`run-tests.sh`](./run-tests.sh) command is most convenient and versatile way to run tests.\
+It leave no traces (outside of `/nix/store`) on the host system.
+
+`run-tests.sh` requires Nix >= 2.10.
+
+### Summary
+```bash
+./run-tests.sh [--scenario|-s <scenario>] [build|vm|debug|container]
+```
+
+See the top of [run-tests.sh](../test/run-tests.sh) for a complete documentation.\
+Test scenarios are defined in [tests.nix](./tests.nix) and [tests.py](tests.py).
+
+### Tutorial
+#### Running tests
+```bash
+# Run the basic set of tests. These tests are also run on the GitHub CI server.
+./run-tests.sh
+
+# Run the test for scenario `regtest`.
+# The test is run via the Nix build system. Successful runs are cached.
+./run-tests.sh -s regtest build
+./run-tests.sh -s regtest # Shorthand, equivalent
+
+# To test a single service, use its name as a scenario.
+./run-tests.sh -s clightning
+
+# When no scenario is specified, scenario `default` is used.
+./run-tests.sh build
+```
+#### Debugging
+```bash
+# Start a shell is inside a test VM. No tests are executed.
+./run-tests.sh -s bitcoind vm
+systemctl status bitcoind
+
+# Run a Python NixOS test shell inside a VM.
+# See https://nixos.org/manual/nixos/stable/#ssec-machine-objects for available commands.
+./run-tests.sh debug
+print(succeed("systemctl status bitcoind"))
+run_test("bitcoind")
+
+# Start a shell in a container node. Requires systemd and root privileges.
+./run-tests.sh container
+
+# In the container shell: Run command in container (with prefix `c`)
+c systemctl status bitcoind
+
+# Explore a single feature
+./run-tests.sh -s electrs container
+
+# Run a command in a container.
+# The container is deleted afterwards.
+./run-tests.sh -s clightning container --run c lightning-cli getinfo
+
+# Define a custom scenario
+./run-tests.sh --scenario '{
+  services.clightning.enable = true;
+  nix-bitcoin.nodeinfo.enable = true;
+}' container --run c nodeinfo
+```
+
+# Running tests with Flakes
+
+Tests can also be accessed via the nix-bitcoin flake:
+
+```bash
+# Build test
+nix build --no-link ..#tests.default
+
+# Run a node in a VM. No tests are executed.
+nix run ..#tests.default.vm
+
+# Run a Python test shell inside a VM node
+nix run ..#tests.default.run -- --debug
+
+# Run a node in a container. Requires extra-container, systemd and root privileges
+nix run ..#tests.default.container
+nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05
+
+# Run a command in a container
+nix run ..#tests.default.container -- --run c nodeinfo
+nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05
+```

test/ci/build.sh

@@ -1,12 +1,14 @@
 #!/usr/bin/env bash
 
 # This script can also be run locally for testing:
-#   scenario=default ./build.sh
+#   ./build.sh <scenario>
 #
 # When variable CIRRUS_CI is unset, this script leaves no persistent traces on the host system.
 
 set -euo pipefail
 
+scenario=$1
+
 if [[ -v CIRRUS_CI ]]; then
     if [[ ! -e /dev/kvm ]]; then
         >&2 echo "No KVM available on VM host."
@@ -16,5 +18,5 @@ if [[ -v CIRRUS_CI ]]; then
     chmod o+rw /dev/kvm
 fi
 
-# shellcheck disable=SC2154
-"${BASH_SOURCE[0]%/*}/../run-tests.sh" --ci --scenario "$scenario"
+cd "${BASH_SOURCE[0]%/*}"
+exec ./build-to-cachix.sh --expr "(builtins.getFlake (toString ../..)).legacyPackages.\${builtins.currentSystem}.tests.$scenario"

test/clightning-replication.nix

@@ -1,22 +1,21 @@
-# You can run this test via `run-tests.sh -s clightningReplication`
+# You can run this test via `run-tests.sh -s clightning-replication`
 
-let
-  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
-in
-import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }:
+makeTestVM: pkgs:
 with pkgs.lib;
 let
-  keyDir = "${nixpkgs}/nixos/tests/initrd-network-ssh";
+  keyDir = pkgs.path + "/nixos/tests/initrd-network-ssh";
   keys = {
-    server = "${keyDir}/ssh_host_ed25519_key";
-    client = "${keyDir}/id_ed25519";
-    serverPub = readFile "${keys.server}.pub";
-    clientPub = readFile "${keys.client}.pub";
+    server = keyDir + "/ssh_host_ed25519_key";
+    client = keyDir + "/id_ed25519";
+    serverPub = readFile (keys.server + ".pub");
+    clientPub = readFile (keys.client + ".pub");
   };
 
   clientBaseConfig = {
     imports = [ ../modules/modules.nix ];
 
+    nixpkgs.pkgs = pkgs;
+
     nix-bitcoin.generateSecrets = true;
 
     services.clightning = {
@@ -29,7 +28,7 @@ let
     };
   };
 in
-{
+makeTestVM {
   name = "clightning-replication";
 
   nodes = let nodes = {
@@ -57,7 +56,9 @@ in
       services.clightning.replication.encrypt = true;
     };
 
-    server = { ... }: {
+    server = {
+      nixpkgs.pkgs = pkgs;
+
       environment.etc."ssh-host-key" = {
         source = keys.server;
         mode = "400";
@@ -131,8 +132,8 @@ in
           # A gocryptfs has been created
           client.succeed("ls /var/backup/clightning/lightningd-db/gocryptfs.conf")
 
-      server.wait_for_unit("sshd.service")
       switch_to_system("replicationRemote")
+      server.wait_for_unit("sshd.service")
       with subtest("remote replication"):
           replica_db = "/var/cache/clightning-replication/sshfs/lightningd.sqlite3"
           client.succeed(f"runuser -u clightning -- ls {replica_db}")
@@ -150,4 +151,4 @@ in
           # A gocryptfs has been created on the server
           server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf")
   '';
-})
+}

test/lib/copy-src.sh

@@ -14,7 +14,7 @@ atExit() {
 trap "atExit" EXIT
 
 # shellcheck disable=SC2154
-rsync -a --delete --exclude='.git*' "$scriptDir/../" "$tmp/src"
+rsync -a --delete "$scriptDir/../" "$tmp/src"
 echo "Copied src"
 
 # shellcheck disable=SC2154

test/lib/create-git-repo.sh

@@ -1,15 +0,0 @@
-# Create and maintain a minimal git repo at the root of the copied src
-(
-  # shellcheck disable=SC2154,SC2164
-  cd "$scriptDir/.."
-  amend=(--amend)
-
-  if [[ ! -e .git ]] || ! git rev-parse HEAD 2>/dev/null; then
-    git init
-    amend=()
-  fi
-  git add .
-  if ! git diff --quiet --cached; then
-    git commit -a "${amend[@]}" -m -
-  fi
-) >/dev/null