Merge fort-nix/nix-bitcoin#504: Add fulcrum module

7d7f2df006 fulcrum: add module (Erik Arvstedt)
edd8bd311c bitcoind-rpc-public-whitelist: add `ping`, `help` (Erik Arvstedt)
f946a7cd9b examples: use `mkDefault` for setting `system.stateVersion` (Erik Arvstedt)
79994d0bd2 tests: increase postgresql startup timeout (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  nixbitcoin:
    ACK 7d7f2df006

Tree-SHA512: 866f0a40bb81ceb2008e6efbd415eeee201b9e56aa5ebc60fd1bb4b6eda300155faaa2e6492d8145b3c6084b1e8ef5ec7d4e0ff694beb328d2e5ed755c9d04b4

README.md

@@ -85,6 +85,7 @@ NixOS modules ([src](modules/modules.nix))
   * [Ride The Lightning](https://github.com/Ride-The-Lightning/RTL): web interface for `lnd` and `clightning`
   * [spark-wallet](https://github.com/shesek/spark-wallet)
   * [electrs](https://github.com/romanz/electrs)
+  * [fulcrum](https://github.com/cculianu/Fulcrum) (see [the module](modules/fulcrum.nix) for a comparison to electrs)
   * [btcpayserver](https://github.com/btcpayserver/btcpayserver)
   * [liquid](https://github.com/elementsproject/elements)
   * [JoinMarket](https://github.com/joinmarket-org/joinmarket-clientserver)

docs/hardware.md

@@ -5,7 +5,7 @@ Hardware requirements
 * Disk space: 500 GB (400GB for Bitcoin blockchain + some room) for an unpruned
   instance of Bitcoin Core.
   * This can be significantly lowered by enabling pruning.
-    Note: Pruning is not supported by `electrs`.
+    Note: Pruning is not supported by `electrs` and `fulcrum`.
 
 Tested low-end hardware includes:
 - [Raspberry Pi 4](https://www.raspberrypi.org/products/raspberry-pi-4-model-b/)

examples/configuration.nix

@@ -122,9 +122,20 @@
   # services.spark-wallet.enable = true;
 
   ### ELECTRS
-  # Set this to enable electrs, an efficient Electrum server implemented in Rust.
+  # Set this to enable electrs, an Electrum server implemented in Rust.
   # services.electrs.enable = true;
 
+  ### FULCRUM
+  # Set this to enable fulcrum, an Electrum server implemented in C++.
+  #
+  # Compared to electrs, fulcrum has higher storage demands but
+  # can serve arbitrary address queries instantly.
+  #
+  # Before enabling fulcrum, and for more info on storage demands,
+  # see the description of option `enable` in ../modules/fulcrum.nix
+  #
+  # services.fulcrum.enable = true;
+
   ### BTCPayServer
   # Set this to enable BTCPayServer, a self-hosted, open-source
   # cryptocurrency payment processor.

examples/deploy-krops.sh

@@ -34,7 +34,7 @@ nix-build --out-link $tmpDir/vm - <<'EOF'
     # warning: Nix search path entry '/nix/var/nix/profiles/per-user/root/channels' does not exist, ignoring
     nix.nixPath = lib.mkForce [];
 
-    system.stateVersion = config.system.nixos.release;
+    system.stateVersion = lib.mkDefault config.system.nixos.release;
   };
 }).config.system.build.vm
 EOF

examples/qemu-vm/minimal-vm.nix

@@ -45,7 +45,7 @@ rec {
         echo o >/proc/sysrq-trigger
       '';
 
-      system.stateVersion = config.system.nixos.release;
+      system.stateVersion = lib.mkDefault config.system.nixos.release;
     };
   }).config.system.build.vm;
 

modules/bitcoind-rpc-public-whitelist.nix

@@ -2,6 +2,8 @@
 [
   "echo"
   "getinfo"
+  "help"
+  "ping"
   "uptime"
   # Blockchain
   "getbestblockhash"

modules/fulcrum.nix

@@ -0,0 +1,139 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  options.services.fulcrum = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enable fulcrum, an Electrum server implemented in C++.
+
+        Compared to electrs, fulcrum has a 3x larger database size but
+        can serve arbitrary address queries instantly.
+
+        fulcrum also enables `txindex` in bitcoind (this is a requirement),
+        which increases the bitcoind datadir size by 8% of the `blocks` size.
+
+        This module disables peering (a distributed list of electrum servers that can
+        be queried by clients), but you can manually enable it via option
+        `extraConfig`.
+      '';
+    };
+    address = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = "Address to listen for RPC connections.";
+    };
+    port = mkOption {
+      type = types.port;
+      default = 50001;
+      description = "Port to listen for RPC connections.";
+    };
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/fulcrum";
+      description = "The data directory for fulcrum.";
+    };
+    extraConfig = mkOption {
+      type = types.lines;
+      default = "";
+      example = ''
+        peering = true
+      '';
+      description = ''
+        Extra lines appended to the configuration file.
+
+        See all available options at
+        https://github.com/cculianu/Fulcrum/blob/master/doc/fulcrum-example-config.conf
+      '';
+    };
+    user = mkOption {
+      type = types.str;
+      default = "fulcrum";
+      description = "The user as which to run fulcrum.";
+    };
+    group = mkOption {
+      type = types.str;
+      default = cfg.user;
+      description = "The group as which to run fulcrum.";
+    };
+    tor.enforce = nbLib.tor.enforce;
+  };
+
+  cfg = config.services.fulcrum;
+  nbLib = config.nix-bitcoin.lib;
+  secretsDir = config.nix-bitcoin.secretsDir;
+  bitcoind = config.services.bitcoind;
+
+  configFile = builtins.toFile "fulcrum.conf" ''
+    datadir = ${cfg.dataDir}
+    tcp = ${cfg.address}:${toString cfg.port}
+
+    bitcoind = ${nbLib.addressWithPort bitcoind.rpc.address bitcoind.rpc.port}
+    rpcuser = ${bitcoind.rpc.users.public.name}
+
+    # Disable logging timestamps
+    ts-format = none
+
+    peering = false
+
+    ${cfg.extraConfig}
+  '';
+in {
+  inherit options;
+
+  config = mkIf cfg.enable {
+    assertions = [
+      { assertion = bitcoind.prune == 0;
+        message = "Fulcrum does not support bitcoind pruning.";
+      }
+      { assertion =
+          !(config.services ? electrs)
+          || !config.services.electrs.enable
+          || config.services.electrs.port != cfg.port;
+        message = ''
+          Fulcrum and Electrs can't both bind to TCP RPC port ${cfg.port}.
+          Change `services.electrs.port` or `services.fulcrum.port`
+          to a port other than ${cfg.port}.
+        '';
+      }
+    ];
+
+    services.bitcoind = {
+      enable = true;
+      txindex = true;
+    };
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
+    ];
+
+    systemd.services.fulcrum = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "bitcoind.service" ];
+      after = [ "bitcoind.service" ];
+      preStart = ''
+        {
+          cat ${configFile}
+          echo "rpcpassword = $(cat ${secretsDir}/bitcoin-rpcpassword-public)"
+        } > '${cfg.dataDir}/fulcrum.conf'
+      '';
+      serviceConfig = nbLib.defaultHardening // {
+        ExecStart = "${config.nix-bitcoin.pkgs.fulcrum}/bin/Fulcrum '${cfg.dataDir}/fulcrum.conf'";
+        User = cfg.user;
+        Group = cfg.group;
+        Restart = "on-failure";
+        RestartSec = "10s";
+        ReadWritePaths = cfg.dataDir;
+      } // nbLib.allowedIPAddresses cfg.tor.enforce;
+    };
+
+    users.users.${cfg.user} = {
+      isSystemUser = true;
+      group = cfg.group;
+      extraGroups = [ "bitcoinrpc-public" ];
+    };
+    users.groups.${cfg.group} = {};
+  };
+}

modules/modules.nix

@@ -21,6 +21,7 @@
     ./lndconnect-onion.nix # Requires onion-addresses.nix
     ./rtl.nix
     ./electrs.nix
+    ./fulcrum.nix
     ./liquid.nix
     ./btcpayserver.nix
     ./joinmarket.nix

modules/netns-isolation.nix

@@ -293,6 +293,10 @@ in {
       clightning-rest = {
         id = 30;
       };
+      fulcrum = {
+        id = 31;
+        connections = [ "bitcoind" ];
+      };
     };
 
     services.bitcoind = {
@@ -323,6 +327,8 @@ in {
 
     services.electrs.address = netns.electrs.address;
 
+    services.fulcrum.address = netns.fulcrum.address;
+
     services.spark-wallet = {
       address = netns.spark-wallet.address;
       extraArgs = "--no-tls";

modules/nodeinfo.nix

@@ -132,6 +132,7 @@ in {
       '';
       clightning-rest = mkInfo "";
       electrs = mkInfo "";
+      fulcrum = mkInfo "";
       spark-wallet = mkInfo "";
       btcpayserver = mkInfo "";
       liquidd = mkInfo "";

modules/presets/enable-tor.nix

@@ -34,6 +34,7 @@ in {
     # but we restrict them to Tor just to be safe.
     #
     electrs = defaultEnforceTor;
+    fulcrum = defaultEnforceTor;
     nbxplorer = defaultEnforceTor;
     rtl = defaultEnforceTor;
     joinmarket = defaultEnforceTor;
@@ -46,6 +47,7 @@ in {
     bitcoind.enable = defaultTrue;
     liquidd.enable = defaultTrue;
     electrs.enable = defaultTrue;
+    fulcrum.enable = defaultTrue;
     spark-wallet.enable = defaultTrue;
     joinmarket-ob-watcher.enable = defaultTrue;
     rtl.enable = defaultTrue;

pkgs/pinned.nix

@@ -14,6 +14,7 @@ pkgs: pkgsUnstable:
   inherit (pkgsUnstable)
     btcpayserver
     clightning
+    fulcrum
     hwi
     lightning-loop
     lnd

test/lib/make-test.nix

@@ -79,7 +79,7 @@ name: testConfig:
        ''))
     ];
 
-    system.stateVersion = config.system.nixos.release;
+    system.stateVersion = lib.mkDefault config.system.nixos.release;
   })).config.system.build.vm;
 
   config = testConfig;

test/tests.nix

@@ -109,6 +109,9 @@ let
 
       tests.electrs = cfg.electrs.enable;
 
+      services.fulcrum.port = 50002;
+      tests.fulcrum = cfg.fulcrum.enable;
+
       tests.liquidd = cfg.liquidd.enable;
       services.liquidd.extraConfig = mkIf config.test.noConnections "connect=0";
 
@@ -141,7 +144,7 @@ let
       '';
 
       # Avoid timeout failures on slow CI nodes
-      systemd.services.postgresql.serviceConfig.TimeoutStartSec = "3min";
+      systemd.services.postgresql.serviceConfig.TimeoutStartSec = "5min";
     }
     (mkIf config.test.features.clightningPlugins {
       services.clightning.plugins = {
@@ -194,6 +197,7 @@ let
       services.lightning-pool.enable = true;
       services.charge-lnd.enable = true;
       services.electrs.enable = true;
+      services.fulcrum.enable = true;
       services.liquidd.enable = true;
       services.btcpayserver.enable = true;
       services.joinmarket.enable = true;
@@ -240,6 +244,7 @@ let
       services.lightning-pool.enable = true;
       services.charge-lnd.enable = true;
       services.electrs.enable = true;
+      services.fulcrum.enable = true;
       services.btcpayserver.enable = true;
       services.joinmarket.enable = true;
     };

test/tests.py

@@ -112,6 +112,11 @@ def _():
             log_has_string("electrs", "waiting for 0 blocks to download")
         )
 
+@test("fulcrum")
+def _():
+    assert_running("fulcrum")
+    machine.wait_until_succeeds(log_has_string("fulcrum", "started ok"))
+
 # Impure: Stops electrs
 # Stop electrs from spamming the test log with 'waiting for 0 blocks to download' messages
 @test("stop-electrs")
@@ -378,33 +383,44 @@ def _():
         else:
             return False
 
+    def get_block_height(ip, port):
+        return (
+            """echo '{"method": "blockchain.headers.subscribe", "id": 0}'"""
+            f" | nc {ip} {port} | head -1 | jq -M .result.height"
+        )
+
     num_blocks = test_data["num_blocks"]
 
     if enabled("electrs"):
         machine.wait_until_succeeds(log_has_string("electrs", "serving Electrum RPC"))
-        get_block_height_cmd = (
+        assert_full_match(get_block_height(ip('electrs'), 50001), f"{num_blocks}\n")
-            """echo '{"method": "blockchain.headers.subscribe", "id": 0, "params": []}'"""
+
-            f" | nc {ip('electrs')} 50001 | head -1 | jq -M .result.height"
+    if enabled("fulcrum"):
-        )
+        machine.wait_until_succeeds(log_has_string("fulcrum", "listening for connections"))
-        assert_full_match(get_block_height_cmd, f"{num_blocks}\n")
+        assert_full_match(get_block_height(ip('fulcrum'), 50002), f"{num_blocks}\n")
+
     if enabled("clightning"):
         machine.wait_until_succeeds(
             f"[[ $(runuser -u operator -- lightning-cli getinfo | jq -M .blockheight) == {num_blocks} ]]"
         )
+
     if enabled("lnd"):
         machine.wait_until_succeeds(
             f"[[ $(runuser -u operator -- lncli getinfo | jq -M .block_height) == {num_blocks} ]]"
         )
+
     if enabled("lightning-loop"):
         machine.wait_until_succeeds(
             log_has_string("lightning-loop", f"Starting event loop at height {num_blocks}")
         )
         succeed("runuser -u operator -- loop getparams")
+
     if enabled("lightning-pool"):
         machine.wait_until_succeeds(
             log_has_string("lightning-pool", "lnd is now fully synced to its chain backend")
         )
         succeed("runuser -u operator -- pool orders list")
+
     if enabled("btcpayserver"):
         machine.wait_until_succeeds(log_has_string("nbxplorer", f"At height: {num_blocks}"))