Merge #259: Adds curated clightning plugins

1d44b99340 add curated clightning plugins (Ian Shipman)
4640821f96 make-test.nix: use writeText (Erik Arvstedt)
5399f73b20 add txzmq python pkg (Erik Arvstedt)
e62e163177 add clightning python pkgs (Erik Arvstedt)
1a16e55237 move python packages to pkgs/python-packages (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK 1d44b99340
  nixbitcoin:
    ACK 1d44b99340

Tree-SHA512: 566d1a606c27042de65d9291a2fbb2ee6866ae2befc43251e35b85cb035ed5aa26d7ef30bcf0364e045bd61d3460840ea5360466015d404febb5299a44c429f7

.travis.yml

@@ -33,6 +33,7 @@ env:
   - PKG=nixops19_09 STABLE=1
   - PKG=joinmarket STABLE=1
   - PKG=joinmarket STABLE=0
+  - PKG=clightning-plugins-all STABLE=1
 script:
   - printf '%s (%s)\n' "$NIX_PATH" "$VER"
   - |

README.md

@@ -91,7 +91,7 @@ By default the `configuration.nix` provides:
 * adds non-root user "operator" which has access to bitcoin-cli and lightning-cli
 
 In `configuration.nix` the user can enable:
-* a clightning hidden service
+* a clightning hidden service with [plugins](https://github.com/lightningd/plugins)
 * [liquid](https://github.com/elementsproject/elements)
 * [lightning charge](https://github.com/ElementsProject/lightning-charge)
 * [nanopos](https://github.com/ElementsProject/nanopos)

docs/usage.md

@@ -338,3 +338,31 @@ See [here](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master
     ```
 
 3. Profit
+
+clightning
+---
+
+## Plugins
+
+There are a number of [plugins](https://github.com/lightningd/plugins) available for clightning. Currently `nix-bitcoin` supports:
+
+- helpme
+- monitor
+- prometheus
+- rebalance
+- summary
+- zmq
+
+You can activate and configure these plugins like so:
+
+```nix
+services.clightning = {
+    enable = true;
+    plugins = {
+        prometheus.enable = true;
+        prometheus.listen = "0.0.0.0:9900";
+    };
+};
+```
+
+Please have a look at the module for a plugin (e.g. [prometheus.nix](../modules/clightning-plugins/prometheus.nix)) to learn its configuration options.

examples/configuration.nix

@@ -38,10 +38,14 @@
   # Enable this module to use clightning, a Lightning Network implementation
   # in C.
   services.clightning.enable = true;
+  # == TOR
   # Enable this option to announce our Tor Hidden Service. By default clightning
   # offers outgoing functionality, but doesn't announce the Tor Hidden Service
   # under which peers can reach us.
   # services.clightning.announce-tor = true;
+  # == Plugins
+  # See ../docs/usage.md for the list of available plugins.
+  # services.clightning.plugins.prometheus.enable = true;
 
   ### LND
   # Uncomment the following line in order to enable lnd, a lightning

modules/clightning-plugins/default.nix

@@ -0,0 +1,27 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.clightning.plugins;
+  pluginPkgs = config.nix-bitcoin.pkgs.clightning-plugins;
+in {
+  imports = [
+    ./prometheus.nix
+    ./summary.nix
+    ./zmq.nix
+  ];
+
+  options.services.clightning.plugins = {
+    helpme.enable = mkEnableOption "Help me (clightning plugin)";
+    monitor.enable = mkEnableOption "Monitor (clightning plugin)";
+    rebalance.enable = mkEnableOption "Rebalance (clightning plugin)";
+  };
+
+  config = {
+    services.clightning.extraConfig = mkMerge [
+      (mkIf cfg.helpme.enable "plugin=${pluginPkgs.helpme.path}")
+      (mkIf cfg.monitor.enable "plugin=${pluginPkgs.monitor.path}")
+      (mkIf cfg.rebalance.enable "plugin=${pluginPkgs.rebalance.path}")
+    ];
+  };
+}

modules/clightning-plugins/prometheus.nix

@@ -0,0 +1,21 @@
+{ config, lib, ... }:
+
+with lib;
+let cfg = config.services.clightning.plugins.prometheus; in
+{
+  options.services.clightning.plugins.prometheus = {
+    enable = mkEnableOption "Prometheus (clightning plugin)";
+    listen = mkOption {
+      type = types.str;
+      default = "0.0.0.0:9750";
+      description = "Address and port to bind to.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${config.nix-bitcoin.pkgs.clightning-plugins.prometheus.path}
+      prometheus-listen=${cfg.listen}
+    '';
+  };
+}

modules/clightning-plugins/summary.nix

@@ -0,0 +1,39 @@
+{ config, lib, ... }:
+
+with lib;
+let cfg = config.services.clightning.plugins.summary; in
+{
+  options.services.clightning.plugins.summary = {
+    enable = mkEnableOption "Summary (clightning plugin)";
+    currency = mkOption {
+      type = types.str;
+      default = "USD";
+      description = "The currency to look up on btcaverage.";
+    };
+    currencyPrefix = mkOption {
+      type = types.str;
+      default = "USD $";
+      description = "The prefix to use for the currency.";
+    };
+    availabilityInterval = mkOption {
+      type = types.int;
+      default = 300;
+      description = "How often in seconds the availability should be calculated.";
+    };
+    availabilityWindow = mkOption {
+      type = types.int;
+      default = 72;
+      description = "How many hours the availability should be averaged over.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${config.nix-bitcoin.pkgs.clightning-plugins.summary.path}
+      summary-currency="${cfg.currency}"
+      summary-currency-prefix="${cfg.currencyPrefix}"
+      summary-availability-interval=${toString cfg.availabilityInterval}
+      summary-availability-window=${toString cfg.availabilityWindow}
+    '';
+  };
+}

modules/clightning-plugins/zmq.nix

@@ -0,0 +1,42 @@
+{ config, lib, ... }:
+
+with lib;
+let
+  cfg = config.services.clightning.plugins.zmq;
+
+  endpoints = [
+    "channel-opened"
+    "connect"
+    "disconnect"
+    "invoice-payment"
+    "warning"
+    "forward-event"
+    "sendpay-success"
+    "sendpay-failure"
+  ];
+
+  mkEndpointOption = name:
+    mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "Endpoint for ${name}";
+    };
+
+  setEndpoint = ep:
+    let value = builtins.getAttr ep cfg; in
+    optionalString (value != null) ''
+      zmq-pub-${ep}=${value}
+    '';
+in
+{
+  options.services.clightning.plugins.zmq = {
+    enable = mkEnableOption "ZMQ (clightning plugin)";
+  } // lib.genAttrs endpoints mkEndpointOption;
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${config.nix-bitcoin.pkgs.clightning-plugins.zmq.path}
+      ${concatStrings (map setEndpoint endpoints)}
+    '';
+  };
+}

modules/modules.nix

@@ -9,6 +9,7 @@
     # Main features
     ./bitcoind.nix
     ./clightning.nix
+    ./clightning-plugins
     ./lightning-charge.nix
     ./nanopos.nix
     ./spark-wallet.nix

pkgs/clightning-plugins/default.nix

@@ -0,0 +1,63 @@
+pkgs: nbPython3Packages:
+
+let
+  inherit (pkgs) lib;
+
+  src = pkgs.fetchFromGitHub {
+    owner = "lightningd";
+    repo = "plugins";
+    rev = "6cd472636926f05a9c472139fabe1ff11c90aa6a";
+    sha256 = "1lisx85vzsfzjhdc6zdz0l6bcrdgg6rp5xbc5jmx93mv8qqg2cns";
+  };
+
+  version = builtins.substring 0 7 src.rev;
+
+  plugins = with nbPython3Packages; {
+    helpme = {};
+    monitor = {};
+    prometheus = {
+      extraPkgs = [ prometheus_client ];
+      patchRequirements = "--replace prometheus-client==0.6.0 prometheus-client==0.8.0";
+    };
+    rebalance = {};
+    summary = {
+      extraPkgs = [ packaging requests ];
+    };
+    zmq = {
+      scriptName = "cl-zmq";
+      extraPkgs = [ twisted txzmq ];
+    };
+  };
+
+  basePkgs = [ nbPython3Packages.pyln-client ];
+
+  mkPlugin = name: plugin: let
+    python = pkgs.python3.withPackages (_: basePkgs ++ (plugin.extraPkgs or []));
+    script = "${plugin.scriptName or name}.py";
+    drv = pkgs.stdenv.mkDerivation {
+      pname = "clightning-plugin-${name}";
+      inherit version;
+
+      buildInputs = [ python ];
+
+      buildCommand = ''
+        cp --no-preserve=mode -r ${src}/${name} $out
+        cd $out
+        ${lib.optionalString (plugin ? patchRequirements) ''
+          substituteInPlace requirements.txt ${plugin.patchRequirements}
+        ''}
+
+        # Check that requirements are met
+        PYTHONPATH=${toString python}/${python.sitePackages} \
+          ${pkgs.python3Packages.pip}/bin/pip install -r requirements.txt --no-cache --no-index
+
+        chmod +x ${script}
+        patchShebangs ${script}
+      '';
+
+      passthru.path = "${drv}/${script}";
+    };
+  in drv;
+
+in
+  builtins.mapAttrs mkPlugin plugins

pkgs/clightning-plugins/get-sha256.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p git
+set -euo pipefail
+
+archive_hash () {
+    repo=$1
+    rev=$2
+    nix-prefetch-url --unpack "https://github.com/${repo}/archive/${rev}.tar.gz" 2> /dev/null | tail -n 1
+}
+
+echo "Fetching latest lightningd/plugins release"
+latest=$(git ls-remote https://github.com/lightningd/plugins master | cut -f 1)
+echo "rev: ${latest}"
+echo "sha256: $(archive_hash lightningd/plugins $latest)"

pkgs/default.nix

@@ -6,18 +6,26 @@ let self = {
   electrs = pkgs.callPackage ./electrs { };
   elementsd = pkgs.callPackage ./elementsd { withGui = false; };
   hwi = pkgs.callPackage ./hwi { };
-  pylightning = pkgs.python3Packages.callPackage ./pylightning { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
-  joinmarket = pkgs.callPackage ./joinmarket { };
+  joinmarket = pkgs.callPackage ./joinmarket { inherit (self) nbPython3Packages; };
   generate-secrets = pkgs.callPackage ./generate-secrets { };
   nixops19_09 = pkgs.callPackage ./nixops { };
   netns-exec = pkgs.callPackage ./netns-exec { };
   lightning-loop = pkgs.callPackage ./lightning-loop { };
   extra-container = pkgs.callPackage ./extra-container { };
+  clightning-plugins = import ./clightning-plugins pkgs self.nbPython3Packages;
+
+  nbPython3Packages = (pkgs.python3.override {
+    packageOverrides = pySelf: super: import ./python-packages self pySelf;
+  }).pkgs;
 
   pinned = import ./pinned.nix;
 
   lib = import ./lib.nix { inherit (pkgs) lib; };
 
   modulesPkgs = self // self.pinned;
+
+  # Used in ../.travis.yml
+  clightning-plugins-all = pkgs.writeText "clightning-plugins"
+    (pkgs.lib.concatMapStringsSep "\n" toString (builtins.attrValues self.clightning-plugins));
 }; in self

pkgs/joinmarket/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchurl, python3, pkgs }:
+{ stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }:
 
 let
   version = "0.7.2";
@@ -7,32 +7,14 @@ let
     sha256 = "03gvs20d2cfzy9x82l6v4c69w0j9mr4p9zj2hpymnb6xs1yq6dr1";
   };
 
-  python = python3.override {
-    packageOverrides = self: super: let
-      joinmarketPkg = pkg: self.callPackage pkg { inherit version src; };
-    in {
-      joinmarketbase = joinmarketPkg ./jmbase;
-      joinmarketclient = joinmarketPkg ./jmclient;
-      joinmarketbitcoin = joinmarketPkg ./jmbitcoin;
-      joinmarketdaemon = joinmarketPkg ./jmdaemon;
-
-      chromalog = self.callPackage ./chromalog {};
-      bencoderpyx = self.callPackage ./bencoderpyx {};
-      coincurve = self.callPackage ./coincurve {};
-      urldecode = self.callPackage ./urldecode {};
-      python-bitcointx = self.callPackage ./python-bitcointx {};
-      secp256k1 = self.callPackage ./secp256k1 {};
-    };
-  };
-
-  runtimePackages = with python.pkgs; [
+  runtimePackages = with nbPython3Packages; [
     joinmarketbase
     joinmarketclient
     joinmarketbitcoin
     joinmarketdaemon
   ];
 
-  pythonEnv = python.withPackages (_: runtimePackages);
+  pythonEnv = python3.withPackages (_: runtimePackages);
 in
 stdenv.mkDerivation {
   pname = "joinmarket";
@@ -62,8 +44,4 @@ stdenv.mkDerivation {
     chmod +x -R $out/bin
     patchShebangs $out/bin
   '';
-
-  passthru = {
-      inherit python runtimePackages pythonEnv;
-  };
 }

pkgs/pylightning/default.nix

@@ -1,11 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi }:
-
-buildPythonPackage rec {
-  pname = "pylightning";
-  version = "0.0.7";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "0anbixqsfk0dsm790yy21f403lwgalxaqlm1s101ifppmxqccgpi";
-  };
-}

pkgs/python-packages/default.nix

@@ -0,0 +1,25 @@
+nbPkgs:
+self:
+let
+  inherit (self) callPackage;
+
+  joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; };
+  clightningPkg = pkg: callPackage pkg { inherit (nbPkgs.pinned) clightning; };
+in {
+  bencoderpyx = callPackage ./bencoderpyx {};
+  coincurve = callPackage ./coincurve {};
+  python-bitcointx = callPackage ./python-bitcointx {};
+  secp256k1 = callPackage ./secp256k1 {};
+  urldecode = callPackage ./urldecode {};
+  chromalog = callPackage ./chromalog {};
+  txzmq = callPackage ./txzmq {};
+
+  joinmarketbase = joinmarketPkg ./jmbase;
+  joinmarketclient = joinmarketPkg ./jmclient;
+  joinmarketbitcoin = joinmarketPkg ./jmbitcoin;
+  joinmarketdaemon = joinmarketPkg ./jmdaemon;
+
+  pyln-client = clightningPkg ./pyln-client;
+  pyln-proto = clightningPkg ./pyln-proto;
+  pylightning = clightningPkg ./pylightning;
+}

pkgs/python-packages/pylightning/default.nix

@@ -0,0 +1,17 @@
+{ buildPythonPackage, clightning, pyln-client }:
+
+buildPythonPackage rec {
+  pname = "pylightning";
+  version = "0.8.0"; # defined in ${src}/contrib/pyln-client/pyln/client/__init__.py
+
+  inherit (clightning) src;
+
+  propagatedBuildInputs = [ pyln-client ];
+
+  postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
+
+  # The clightning source contains pyln-client 0.8.0
+  postPatch = ''
+    substituteInPlace requirements.txt --replace pyln-client==0.7.3 pyln-client==0.8.0
+  '';
+}

pkgs/python-packages/pyln-client/default.nix

@@ -0,0 +1,12 @@
+{ buildPythonPackage, clightning, recommonmark }:
+
+buildPythonPackage rec {
+  pname = "pyln-client";
+  version = "0.8.0"; # defined in ${src}/contrib/pyln-client/pyln/client/__init__.py
+
+  inherit (clightning) src;
+
+  propagatedBuildInputs = [ recommonmark ];
+
+  postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
+}

pkgs/python-packages/pyln-proto/default.nix

@@ -0,0 +1,31 @@
+{ buildPythonPackage, clightning
+, bitstring
+, cryptography
+, coincurve
+, base58
+, mypy
+}:
+
+buildPythonPackage rec {
+  pname = "pyln-proto";
+  version = "0.8.4"; # defined in ${src}/contrib/pyln-proto/setup.py
+
+  inherit (clightning) src;
+
+  propagatedBuildInputs = [
+    bitstring
+    cryptography
+    coincurve
+    base58
+    mypy
+  ];
+
+  postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
+
+  postPatch = ''
+    substituteInPlace requirements.txt \
+      --replace base58==1.0.2 base58==2.0.1 \
+      --replace bitstring==3.1.6 bitstring==3.1.5 \
+      --replace cryptography==2.8 cryptography==3.1
+  '';
+}

pkgs/python-packages/txzmq/default.nix

@@ -0,0 +1,28 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, twisted
+, pyzmq
+}:
+
+buildPythonPackage rec {
+  pname = "txzmq";
+  version = "0.8.2";
+
+  src = fetchPypi {
+    pname = "txZMQ";
+    inherit version;
+    sha256 = "07a9a480e58d4d732eef9efd7e264f2348cbf27ee82b338ec818a8504006e1c0";
+  };
+
+  propagatedBuildInputs = [
+    twisted
+    pyzmq
+  ];
+
+  meta = with lib; {
+    description = "Twisted bindings for ZeroMQ";
+    homepage = https://github.com/smira/txZMQ;
+    license = licenses.gpl2;
+  };
+}

test/lib/make-test-vm.nix

@@ -3,7 +3,7 @@ testArgs:
 let
   pkgs = import <nixpkgs> { config = {}; overlays = []; };
 
-  test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") testArgs;
+  test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs);
 
   fixedTest = { system ? builtins.currentSystem, ... }@args:
     let

test/lib/make-test.nix

@@ -1,7 +1,7 @@
 scenario: testConfig:
 
 {
-  vm = import ./make-test-vm.nix {
+  vm = import ./make-test-vm.nix (pkgs: {
     name = "nix-bitcoin-${scenario}";
 
     machine = {
@@ -16,7 +16,7 @@ scenario: testConfig:
         data = cfg.test.data;
         tests = cfg.tests;
       };
-      dataFile = builtins.toFile "test-data" (builtins.toJSON data);
+      dataFile = pkgs.writeText "test-data" (builtins.toJSON data);
       initData = ''
         import json
 
@@ -37,7 +37,7 @@ scenario: testConfig:
               run_tests()
         ''
       ];
-  };
+  });
 
   container = {
     # The container name has a 11 char length limit

test/tests.nix

@@ -21,7 +21,11 @@ let testEnv = rec {
       }
     ];
 
-    config = {
+    options.test.features = {
+      clightningPlugins = mkEnableOption "all clightning plugins";
+    };
+
+    config = mkMerge [{
       tests.bitcoind = cfg.bitcoind.enable;
       services.bitcoind = {
         enable = true;
@@ -31,6 +35,11 @@ let testEnv = rec {
       tests.clightning = cfg.clightning.enable;
       # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s.
       services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns";
+      test.data.clightning-plugins = let
+        plugins = config.services.clightning.plugins;
+        enabled = builtins.filter (plugin: plugins.${plugin}.enable) (builtins.attrNames plugins);
+        pluginPkgs = config.nix-bitcoin.pkgs.clightning-plugins;
+      in map (plugin: pluginPkgs.${plugin}.path) enabled;
 
       tests.spark-wallet = cfg.spark-wallet.enable;
 
@@ -67,7 +76,28 @@ let testEnv = rec {
       systemd.services.generate-secrets.postStart = mkIfTest "security" ''
         install -o nobody -g nogroup -m777 <(:) /secrets/dummy
       '';
-    };
+    }
+    (mkIf config.test.features.clightningPlugins {
+      services.clightning.plugins = {
+        helpme.enable = true;
+        monitor.enable = true;
+        prometheus.enable = true;
+        rebalance.enable = true;
+        summary.enable = true;
+        zmq = let tcpEndpoint = "tcp://127.0.0.1:5501"; in {
+          enable = true;
+          channel-opened = tcpEndpoint;
+          connect = tcpEndpoint;
+          disconnect = tcpEndpoint;
+          invoice-payment = tcpEndpoint;
+          warning = tcpEndpoint;
+          forward-event = tcpEndpoint;
+          sendpay-success = tcpEndpoint;
+          sendpay-failure = tcpEndpoint;
+        };
+      };
+    })
+    ];
   };
 
   scenarios = {
@@ -80,6 +110,7 @@ let testEnv = rec {
       tests.security = true;
 
       services.clightning.enable = true;
+      test.features.clightningPlugins = true;
       services.spark-wallet.enable = true;
       services.lightning-charge.enable = true;
       services.nanopos.enable = true;
@@ -120,6 +151,7 @@ let testEnv = rec {
     regtest = {
       imports = [ scenarios.regtestBase ];
       services.clightning.enable = true;
+      test.features.clightningPlugins = true;
       services.spark-wallet.enable = true;
       services.lnd.enable = true;
       services.lightning-loop.enable = true;

test/tests.py

@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import json
 
 
 def succeed(*cmds):
@@ -138,6 +139,20 @@ def _():
 def _():
     assert_running("clightning")
     assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+    if test_data["clightning-plugins"]:
+        plugin_list = succeed("lightning-cli plugin list")
+        plugins = json.loads(plugin_list)["plugins"]
+        active = set(plugin["name"] for plugin in plugins if plugin["active"])
+        failed = set(test_data["clightning-plugins"]).difference(active)
+        if failed:
+            raise Exception(
+                f"The following clightning plugins are inactive:\n{failed}.\n\n"
+                f"Output of 'lightning-cli plugin list':\n{plugin_list}"
+            )
+        else:
+            log.log("Active clightning plugins:")
+            for p in test_data["clightning-plugins"]:
+                log.log(os.path.basename(p))
 
 
 @test("lnd")