Merge #313: Lightning pool

eb21012745 pool: add pkg, module & tests (kon)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK eb21012745
  nixbitcoin:
    ACK eb21012745

Tree-SHA512: 0083b5f7bcc803c475cce3414923f43df0b80d6b85aaa54d6b5da637f2aaa7ea6e7525a5eff41be49921bed0214972e356c79b6e4086468c13716c925ec6f8b0

README.md

@@ -69,6 +69,7 @@ NixOS modules
   * [btcpayserver](https://github.com/btcpayserver/btcpayserver)
   * [liquid](https://github.com/elementsproject/elements)
   * [Lightning Loop](https://github.com/lightninglabs/loop)
+  * [Lightning Pool](https://github.com/lightninglabs/pool)
   * [JoinMarket](https://github.com/joinmarket-org/joinmarket-clientserver)
     * [JoinMarket Orderbook Watcher](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/orderbook.md)
   * [recurring-donations](modules/recurring-donations.nix): for periodic lightning payments

examples/configuration.nix

@@ -159,6 +159,17 @@
   # Automatically enables lnd.
   # services.lightning-loop.enable = true;
 
+  ### lightning-pool
+  # Enable this module to use Lightning Lab's non-custodial batched uniform
+  # clearing-price auction for Lightning Channel Leases.
+  # Use the `pool` command to interact with the lightning-pool service.
+  # Automatically enables lnd.
+  # services.lightning-pool.enable = true;
+  #
+  # lightning-pool requires that lnd has a publicly reachable address.
+  # Set this to create a public onion service for lnd.
+  # nix-bitcoin.onionServices.lnd.public = true;
+
   ### Backups
   # Enable this module to use nix-bitcoin's own backups module. By default, it
   # uses duplicity to incrementally back up all important files in /var/lib to

modules/lightning-pool.nix

@@ -0,0 +1,108 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.lightning-pool;
+  nbLib = config.nix-bitcoin.lib;
+  secretsDir = config.nix-bitcoin.secretsDir;
+  network = config.services.bitcoind.network;
+  rpclisten = "${cfg.rpcAddress}:${toString cfg.rpcPort}";
+  configFile = builtins.toFile "pool.conf" ''
+    rpclisten=${rpclisten}
+    restlisten=${cfg.restAddress}:${toString cfg.restPort}
+    ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
+
+    lnd.host=${config.services.lnd.rpcAddress}:${toString config.services.lnd.rpcPort}
+    lnd.macaroondir=${config.services.lnd.networkDir}
+    lnd.tlspath=${secretsDir}/lnd-cert
+
+    ${cfg.extraConfig}
+  '';
+in {
+  options.services.lightning-pool = {
+    enable = mkEnableOption "lightning-pool";
+    rpcAddress = mkOption {
+       type = types.str;
+       default = "localhost";
+       description = "Address to listen for gRPC connections.";
+    };
+    rpcPort = mkOption {
+       type = types.port;
+       default = 12010;
+       description = "Port to listen for gRPC connections.";
+    };
+    restAddress = mkOption {
+       type = types.str;
+       default = cfg.rpcAddress;
+       description = "Address to listen for REST connections.";
+    };
+    restPort = mkOption {
+       type = types.port;
+       default = 8281;
+       description = "Port to listen for REST connections.";
+    };
+    package = mkOption {
+      type = types.package;
+      default = config.nix-bitcoin.pkgs.lightning-pool;
+      description = "The package providing lightning-pool binaries.";
+    };
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/lightning-pool";
+      description = "The data directory for lightning-pool.";
+    };
+    proxy = mkOption {
+      type = types.nullOr types.str;
+      default = if cfg.enforceTor then config.services.tor.client.socksListenAddress else null;
+      description = "host:port of SOCKS5 proxy for connnecting to the pool auction server.";
+    };
+    extraConfig = mkOption {
+      type = types.lines;
+      default = "";
+      example = ''
+        debuglevel=trace
+      '';
+      description = "Extra lines appended to the configuration file.";
+    };
+    cli = mkOption {
+      default = pkgs.writeScriptBin "pool" ''
+        exec ${cfg.package}/bin/pool \
+          --rpcserver ${rpclisten} \
+          --network ${network} \
+          --basedir '${cfg.dataDir}' "$@"
+      '';
+      description = "Binary to connect with the lightning-pool instance.";
+    };
+    enforceTor = nbLib.enforceTor;
+  };
+
+  config = mkIf cfg.enable {
+    services.lnd.enable = true;
+
+    environment.systemPackages = [ cfg.package (hiPrio cfg.cli) ];
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0770 lnd lnd - -"
+    ];
+
+    systemd.services.lightning-pool = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "lnd.service" ];
+      after = [ "lnd.service" ];
+      preStart = ''
+        mkdir -p '${cfg.dataDir}/${network}'
+        ln -sfn ${configFile} '${cfg.dataDir}/${network}/poold.conf'
+      '';
+      serviceConfig = nbLib.defaultHardening // {
+        ExecStart = "${cfg.package}/bin/poold --basedir='${cfg.dataDir}' --network=${network}";
+        User = "lnd";
+        Restart = "on-failure";
+        RestartSec = "10s";
+        ReadWritePaths = cfg.dataDir;
+      } // (if cfg.enforceTor
+            then nbLib.allowTor
+            else nbLib.allowAnyIP);
+    };
+  };
+}

modules/modules.nix

@@ -15,6 +15,7 @@ with lib;
     ./lnd.nix
     ./lnd-rest-onion-service.nix
     ./lightning-loop.nix
+    ./lightning-pool.nix
     ./btcpayserver.nix
     ./electrs.nix
     ./liquid.nix

modules/netns-isolation.nix

@@ -245,6 +245,10 @@ in {
       joinmarket-ob-watcher = {
         id = 26;
       };
+      lightning-pool = {
+        id = 27;
+        connections = [ "lnd" ];
+      };
     };
 
     services.bitcoind = {
@@ -290,6 +294,8 @@ in {
     systemd.services.joinmarket-yieldgenerator.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-joinmarket";
 
     services.joinmarket-ob-watcher.address = netns.joinmarket-ob-watcher.address;
+
+    services.lightning-pool.rpcAddress = netns.lightning-pool.address;
   }
   ]);
 }

modules/presets/enable-tor.nix

@@ -20,6 +20,7 @@ in {
     nbxplorer.enforceTor = true;
     spark-wallet.enforceTor = true;
     recurring-donations.enforceTor = true;
+    lightning-pool.enforceTor = true;
   };
 
   # Add onion services for incoming connections

pkgs/default.nix

@@ -10,6 +10,7 @@ let self = {
   nixops19_09 = pkgs.callPackage ./nixops { };
   netns-exec = pkgs.callPackage ./netns-exec { };
   lightning-loop = pkgs.callPackage ./lightning-loop { };
+  lightning-pool = pkgs.callPackage ./lightning-pool { };
   extra-container = pkgs.callPackage ./extra-container { };
   clightning-plugins = import ./clightning-plugins pkgs self.nbPython3Packages;
   clboss = pkgs.callPackage ./clboss { };

pkgs/lightning-pool/default.nix

@@ -0,0 +1,25 @@
+{ pkgs, buildGoModule, fetchurl, lib }:
+
+buildGoModule rec {
+  pname = "lightning-pool";
+  version = "0.4.4-alpha";
+
+  src = fetchurl {
+    url = "https://github.com/lightninglabs/pool/archive/v${version}.tar.gz";
+    # Use ./get-sha256.sh to fetch latest (verified) sha256
+    sha256 = "c50f3b10a4fa7ce1afd6a00fd6c44474f44c8c773de34261e62ae805540ab140";
+  };
+
+  subPackages = [ "cmd/pool" "cmd/poold" ];
+
+  vendorSha256 = "190qy3cz18ipv8ilpqhbaaxfi9j2isxpwhagzzspa3pwcpssrv52";
+
+  meta = with lib; {
+    description = ''
+      A non-custodial batched uniform clearing-price auction for Lightning Channel Leases (LCL)
+    '';
+    homepage = "https://github.com/lightninglabs/pool";
+    license = lib.licenses.mit;
+    maintainers = with maintainers; [ sputn1ck ];
+  };
+}

pkgs/lightning-pool/get-sha256.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p git gnupg
+set -euo pipefail
+
+TMPDIR="$(mktemp -d -p /tmp)"
+trap "rm -rf $TMPDIR" EXIT
+cd $TMPDIR
+
+echo "Fetching latest release"
+git clone https://github.com/lightninglabs/pool 2> /dev/null
+cd pool
+latest=$(git describe --tags `git rev-list --tags --max-count=1`)
+echo "Latest release is ${latest}"
+
+# GPG verification
+export GNUPGHOME=$TMPDIR
+echo "Fetching Olaoluwa Osuntokun's key"
+gpg --keyserver hkps://keys.openpgp.org --recv-keys 60A1FA7DA5BFF08BDCBBE7903BBD59E99B280306 2> /dev/null
+echo "Fetching Oliver Gugger's key"
+gpg --keyserver hkps://keys.openpgp.org --recv-keys F4FC70F07310028424EFC20A8E4256593F177720 2> /dev/null
+
+echo "Verifying latest release"
+git verify-tag ${latest}
+
+echo "tag: ${latest}"
+# The prefix option is necessary because GitHub prefixes the archive contents in this format
+echo "sha256: $(git archive --format tar.gz --prefix=pool-${latest//v}/ ${latest} | sha256sum | cut -d\  -f1)"

test/pkgs-unstable.nix

@@ -7,6 +7,7 @@ let
     hwi
     joinmarket
     lightning-loop
+    lightning-pool
 
     ## elementsd fails with error
     # test/key_properties.cpp:16:10: fatal error: rapidcheck/boost_test.h: No such file or directory

test/tests.nix

@@ -50,6 +50,9 @@ let testEnv = rec {
 
       tests.lightning-loop = cfg.lightning-loop.enable;
 
+      tests.lightning-pool = cfg.lightning-pool.enable;
+      nix-bitcoin.onionServices.lnd.public = true;
+
       tests.electrs = cfg.electrs.enable;
 
       tests.liquidd = cfg.liquidd.enable;
@@ -120,6 +123,7 @@ let testEnv = rec {
       services.lnd.enable = true;
       services.lnd.restOnionService.enable = true;
       services.lightning-loop.enable = true;
+      services.lightning-pool.enable = true;
       services.electrs.enable = true;
       services.liquidd.enable = true;
       services.btcpayserver.enable = true;
@@ -161,6 +165,7 @@ let testEnv = rec {
       services.spark-wallet.enable = true;
       services.lnd.enable = true;
       services.lightning-loop.enable = true;
+      services.lightning-pool.enable = true;
       services.electrs.enable = true;
       services.btcpayserver.enable = true;
       services.joinmarket.enable = true;
@@ -202,6 +207,12 @@ let testEnv = rec {
         server.host=localhost
       '';
 
+      # lightning-pool contains no builtin auction server for regtest.
+      # Add a dummy definition
+      services.lightning-pool.extraConfig = ''
+        auctionserver=localhost
+      '';
+
       # Needs wallet support which is unavailable for regtest
       services.joinmarket.yieldgenerator.enable = mkForce false;
     };

test/tests.py

@@ -185,6 +185,20 @@ def _():
     )
 
 
+@test("lightning-pool")
+def _():
+    assert_running("lightning-pool")
+    assert_matches("su operator -c 'pool --version'", "version")
+    # Check that lightning-pool fails with the right error, making sure
+    # lightning-pool can connect to lnd
+    machine.wait_until_succeeds(
+        log_has_string(
+            "lightning-pool",
+            "Waiting for lnd to be fully synced to its chain backend, this might take a while",
+        )
+    )
+
+
 @test("btcpayserver")
 def _():
     assert_running("nbxplorer")
@@ -333,7 +347,9 @@ def _():
     pre_restart = succeed("date +%s.%6N").rstrip()
 
     # Sanity-check system by restarting all services
-    succeed("systemctl restart bitcoind clightning lnd lightning-loop spark-wallet liquidd")
+    succeed(
+        "systemctl restart bitcoind clightning lnd lightning-loop lightning-pool spark-wallet liquidd"
+    )
 
     # Now that the bitcoind restart triggered a banlist import restart, check that
     # re-importing already banned addresses works
@@ -365,6 +381,11 @@ def _():
             log_has_string("lightning-loop", "Starting event loop at height 10")
         )
         succeed("runuser -u operator -- loop getparams")
+    if "lightning-pool" in enabled_tests:
+        machine.wait_until_succeeds(
+            log_has_string("lightning-pool", "lnd is now fully synced to its chain backend")
+        )
+        succeed("runuser -u operator -- pool orders list")
 
 
 if "netns-isolation" in enabled_tests: