Merge #259: Adds curated clightning plugins

1d44b99340 add curated clightning plugins (Ian Shipman)
4640821f96 make-test.nix: use writeText (Erik Arvstedt)
5399f73b20 add txzmq python pkg (Erik Arvstedt)
e62e163177 add clightning python pkgs (Erik Arvstedt)
1a16e55237 move python packages to pkgs/python-packages (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK 1d44b99340
  nixbitcoin:
    ACK 1d44b99340

Tree-SHA512: 566d1a606c27042de65d9291a2fbb2ee6866ae2befc43251e35b85cb035ed5aa26d7ef30bcf0364e045bd61d3460840ea5360466015d404febb5299a44c429f7
This commit is contained in:
Jonas Nick 2020-11-22 20:06:12 +00:00
commit fdc338e4a3
No known key found for this signature in database
GPG Key ID: 4861DBF262123605
34 changed files with 420 additions and 45 deletions

View File

@ -33,6 +33,7 @@ env:
- PKG=nixops19_09 STABLE=1 - PKG=nixops19_09 STABLE=1
- PKG=joinmarket STABLE=1 - PKG=joinmarket STABLE=1
- PKG=joinmarket STABLE=0 - PKG=joinmarket STABLE=0
- PKG=clightning-plugins-all STABLE=1
script: script:
- printf '%s (%s)\n' "$NIX_PATH" "$VER" - printf '%s (%s)\n' "$NIX_PATH" "$VER"
- | - |

View File

@ -91,7 +91,7 @@ By default the `configuration.nix` provides:
* adds non-root user "operator" which has access to bitcoin-cli and lightning-cli * adds non-root user "operator" which has access to bitcoin-cli and lightning-cli
In `configuration.nix` the user can enable: In `configuration.nix` the user can enable:
* a clightning hidden service * a clightning hidden service with [plugins](https://github.com/lightningd/plugins)
* [liquid](https://github.com/elementsproject/elements) * [liquid](https://github.com/elementsproject/elements)
* [lightning charge](https://github.com/ElementsProject/lightning-charge) * [lightning charge](https://github.com/ElementsProject/lightning-charge)
* [nanopos](https://github.com/ElementsProject/nanopos) * [nanopos](https://github.com/ElementsProject/nanopos)

View File

@ -338,3 +338,31 @@ See [here](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master
``` ```
3. Profit 3. Profit
clightning
---
## Plugins
There are a number of [plugins](https://github.com/lightningd/plugins) available for clightning. Currently `nix-bitcoin` supports:
- helpme
- monitor
- prometheus
- rebalance
- summary
- zmq
You can activate and configure these plugins like so:
```nix
services.clightning = {
enable = true;
plugins = {
prometheus.enable = true;
prometheus.listen = "0.0.0.0:9900";
};
};
```
Please have a look at the module for a plugin (e.g. [prometheus.nix](../modules/clightning-plugins/prometheus.nix)) to learn its configuration options.

View File

@ -38,10 +38,14 @@
# Enable this module to use clightning, a Lightning Network implementation # Enable this module to use clightning, a Lightning Network implementation
# in C. # in C.
services.clightning.enable = true; services.clightning.enable = true;
# == TOR
# Enable this option to announce our Tor Hidden Service. By default clightning # Enable this option to announce our Tor Hidden Service. By default clightning
# offers outgoing functionality, but doesn't announce the Tor Hidden Service # offers outgoing functionality, but doesn't announce the Tor Hidden Service
# under which peers can reach us. # under which peers can reach us.
# services.clightning.announce-tor = true; # services.clightning.announce-tor = true;
# == Plugins
# See ../docs/usage.md for the list of available plugins.
# services.clightning.plugins.prometheus.enable = true;
### LND ### LND
# Uncomment the following line in order to enable lnd, a lightning # Uncomment the following line in order to enable lnd, a lightning

View File

@ -0,0 +1,27 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.clightning.plugins;
pluginPkgs = config.nix-bitcoin.pkgs.clightning-plugins;
in {
imports = [
./prometheus.nix
./summary.nix
./zmq.nix
];
options.services.clightning.plugins = {
helpme.enable = mkEnableOption "Help me (clightning plugin)";
monitor.enable = mkEnableOption "Monitor (clightning plugin)";
rebalance.enable = mkEnableOption "Rebalance (clightning plugin)";
};
config = {
services.clightning.extraConfig = mkMerge [
(mkIf cfg.helpme.enable "plugin=${pluginPkgs.helpme.path}")
(mkIf cfg.monitor.enable "plugin=${pluginPkgs.monitor.path}")
(mkIf cfg.rebalance.enable "plugin=${pluginPkgs.rebalance.path}")
];
};
}

View File

@ -0,0 +1,21 @@
{ config, lib, ... }:
with lib;
let cfg = config.services.clightning.plugins.prometheus; in
{
options.services.clightning.plugins.prometheus = {
enable = mkEnableOption "Prometheus (clightning plugin)";
listen = mkOption {
type = types.str;
default = "0.0.0.0:9750";
description = "Address and port to bind to.";
};
};
config = mkIf cfg.enable {
services.clightning.extraConfig = ''
plugin=${config.nix-bitcoin.pkgs.clightning-plugins.prometheus.path}
prometheus-listen=${cfg.listen}
'';
};
}

View File

@ -0,0 +1,39 @@
{ config, lib, ... }:
with lib;
let cfg = config.services.clightning.plugins.summary; in
{
options.services.clightning.plugins.summary = {
enable = mkEnableOption "Summary (clightning plugin)";
currency = mkOption {
type = types.str;
default = "USD";
description = "The currency to look up on btcaverage.";
};
currencyPrefix = mkOption {
type = types.str;
default = "USD $";
description = "The prefix to use for the currency.";
};
availabilityInterval = mkOption {
type = types.int;
default = 300;
description = "How often in seconds the availability should be calculated.";
};
availabilityWindow = mkOption {
type = types.int;
default = 72;
description = "How many hours the availability should be averaged over.";
};
};
config = mkIf cfg.enable {
services.clightning.extraConfig = ''
plugin=${config.nix-bitcoin.pkgs.clightning-plugins.summary.path}
summary-currency="${cfg.currency}"
summary-currency-prefix="${cfg.currencyPrefix}"
summary-availability-interval=${toString cfg.availabilityInterval}
summary-availability-window=${toString cfg.availabilityWindow}
'';
};
}

View File

@ -0,0 +1,42 @@
{ config, lib, ... }:
with lib;
let
cfg = config.services.clightning.plugins.zmq;
endpoints = [
"channel-opened"
"connect"
"disconnect"
"invoice-payment"
"warning"
"forward-event"
"sendpay-success"
"sendpay-failure"
];
mkEndpointOption = name:
mkOption {
type = types.nullOr types.str;
default = null;
description = "Endpoint for ${name}";
};
setEndpoint = ep:
let value = builtins.getAttr ep cfg; in
optionalString (value != null) ''
zmq-pub-${ep}=${value}
'';
in
{
options.services.clightning.plugins.zmq = {
enable = mkEnableOption "ZMQ (clightning plugin)";
} // lib.genAttrs endpoints mkEndpointOption;
config = mkIf cfg.enable {
services.clightning.extraConfig = ''
plugin=${config.nix-bitcoin.pkgs.clightning-plugins.zmq.path}
${concatStrings (map setEndpoint endpoints)}
'';
};
}

View File

@ -9,6 +9,7 @@
# Main features # Main features
./bitcoind.nix ./bitcoind.nix
./clightning.nix ./clightning.nix
./clightning-plugins
./lightning-charge.nix ./lightning-charge.nix
./nanopos.nix ./nanopos.nix
./spark-wallet.nix ./spark-wallet.nix

View File

@ -0,0 +1,63 @@
pkgs: nbPython3Packages:
let
inherit (pkgs) lib;
src = pkgs.fetchFromGitHub {
owner = "lightningd";
repo = "plugins";
rev = "6cd472636926f05a9c472139fabe1ff11c90aa6a";
sha256 = "1lisx85vzsfzjhdc6zdz0l6bcrdgg6rp5xbc5jmx93mv8qqg2cns";
};
version = builtins.substring 0 7 src.rev;
plugins = with nbPython3Packages; {
helpme = {};
monitor = {};
prometheus = {
extraPkgs = [ prometheus_client ];
patchRequirements = "--replace prometheus-client==0.6.0 prometheus-client==0.8.0";
};
rebalance = {};
summary = {
extraPkgs = [ packaging requests ];
};
zmq = {
scriptName = "cl-zmq";
extraPkgs = [ twisted txzmq ];
};
};
basePkgs = [ nbPython3Packages.pyln-client ];
mkPlugin = name: plugin: let
python = pkgs.python3.withPackages (_: basePkgs ++ (plugin.extraPkgs or []));
script = "${plugin.scriptName or name}.py";
drv = pkgs.stdenv.mkDerivation {
pname = "clightning-plugin-${name}";
inherit version;
buildInputs = [ python ];
buildCommand = ''
cp --no-preserve=mode -r ${src}/${name} $out
cd $out
${lib.optionalString (plugin ? patchRequirements) ''
substituteInPlace requirements.txt ${plugin.patchRequirements}
''}
# Check that requirements are met
PYTHONPATH=${toString python}/${python.sitePackages} \
${pkgs.python3Packages.pip}/bin/pip install -r requirements.txt --no-cache --no-index
chmod +x ${script}
patchShebangs ${script}
'';
passthru.path = "${drv}/${script}";
};
in drv;
in
builtins.mapAttrs mkPlugin plugins

View File

@ -0,0 +1,14 @@
#! /usr/bin/env nix-shell
#! nix-shell -i bash -p git
set -euo pipefail
archive_hash () {
repo=$1
rev=$2
nix-prefetch-url --unpack "https://github.com/${repo}/archive/${rev}.tar.gz" 2> /dev/null | tail -n 1
}
echo "Fetching latest lightningd/plugins release"
latest=$(git ls-remote https://github.com/lightningd/plugins master | cut -f 1)
echo "rev: ${latest}"
echo "sha256: $(archive_hash lightningd/plugins $latest)"

View File

@ -6,18 +6,26 @@ let self = {
electrs = pkgs.callPackage ./electrs { }; electrs = pkgs.callPackage ./electrs { };
elementsd = pkgs.callPackage ./elementsd { withGui = false; }; elementsd = pkgs.callPackage ./elementsd { withGui = false; };
hwi = pkgs.callPackage ./hwi { }; hwi = pkgs.callPackage ./hwi { };
pylightning = pkgs.python3Packages.callPackage ./pylightning { };
liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { }; liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
joinmarket = pkgs.callPackage ./joinmarket { }; joinmarket = pkgs.callPackage ./joinmarket { inherit (self) nbPython3Packages; };
generate-secrets = pkgs.callPackage ./generate-secrets { }; generate-secrets = pkgs.callPackage ./generate-secrets { };
nixops19_09 = pkgs.callPackage ./nixops { }; nixops19_09 = pkgs.callPackage ./nixops { };
netns-exec = pkgs.callPackage ./netns-exec { }; netns-exec = pkgs.callPackage ./netns-exec { };
lightning-loop = pkgs.callPackage ./lightning-loop { }; lightning-loop = pkgs.callPackage ./lightning-loop { };
extra-container = pkgs.callPackage ./extra-container { }; extra-container = pkgs.callPackage ./extra-container { };
clightning-plugins = import ./clightning-plugins pkgs self.nbPython3Packages;
nbPython3Packages = (pkgs.python3.override {
packageOverrides = pySelf: super: import ./python-packages self pySelf;
}).pkgs;
pinned = import ./pinned.nix; pinned = import ./pinned.nix;
lib = import ./lib.nix { inherit (pkgs) lib; }; lib = import ./lib.nix { inherit (pkgs) lib; };
modulesPkgs = self // self.pinned; modulesPkgs = self // self.pinned;
# Used in ../.travis.yml
clightning-plugins-all = pkgs.writeText "clightning-plugins"
(pkgs.lib.concatMapStringsSep "\n" toString (builtins.attrValues self.clightning-plugins));
}; in self }; in self

View File

@ -1,4 +1,4 @@
{ stdenv, fetchurl, python3, pkgs }: { stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }:
let let
version = "0.7.2"; version = "0.7.2";
@ -7,32 +7,14 @@ let
sha256 = "03gvs20d2cfzy9x82l6v4c69w0j9mr4p9zj2hpymnb6xs1yq6dr1"; sha256 = "03gvs20d2cfzy9x82l6v4c69w0j9mr4p9zj2hpymnb6xs1yq6dr1";
}; };
python = python3.override { runtimePackages = with nbPython3Packages; [
packageOverrides = self: super: let
joinmarketPkg = pkg: self.callPackage pkg { inherit version src; };
in {
joinmarketbase = joinmarketPkg ./jmbase;
joinmarketclient = joinmarketPkg ./jmclient;
joinmarketbitcoin = joinmarketPkg ./jmbitcoin;
joinmarketdaemon = joinmarketPkg ./jmdaemon;
chromalog = self.callPackage ./chromalog {};
bencoderpyx = self.callPackage ./bencoderpyx {};
coincurve = self.callPackage ./coincurve {};
urldecode = self.callPackage ./urldecode {};
python-bitcointx = self.callPackage ./python-bitcointx {};
secp256k1 = self.callPackage ./secp256k1 {};
};
};
runtimePackages = with python.pkgs; [
joinmarketbase joinmarketbase
joinmarketclient joinmarketclient
joinmarketbitcoin joinmarketbitcoin
joinmarketdaemon joinmarketdaemon
]; ];
pythonEnv = python.withPackages (_: runtimePackages); pythonEnv = python3.withPackages (_: runtimePackages);
in in
stdenv.mkDerivation { stdenv.mkDerivation {
pname = "joinmarket"; pname = "joinmarket";
@ -62,8 +44,4 @@ stdenv.mkDerivation {
chmod +x -R $out/bin chmod +x -R $out/bin
patchShebangs $out/bin patchShebangs $out/bin
''; '';
passthru = {
inherit python runtimePackages pythonEnv;
};
} }

View File

@ -1,11 +0,0 @@
{ lib, buildPythonPackage, fetchPypi }:
buildPythonPackage rec {
pname = "pylightning";
version = "0.0.7";
src = fetchPypi {
inherit pname version;
sha256 = "0anbixqsfk0dsm790yy21f403lwgalxaqlm1s101ifppmxqccgpi";
};
}

View File

@ -0,0 +1,25 @@
nbPkgs:
self:
let
inherit (self) callPackage;
joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; };
clightningPkg = pkg: callPackage pkg { inherit (nbPkgs.pinned) clightning; };
in {
bencoderpyx = callPackage ./bencoderpyx {};
coincurve = callPackage ./coincurve {};
python-bitcointx = callPackage ./python-bitcointx {};
secp256k1 = callPackage ./secp256k1 {};
urldecode = callPackage ./urldecode {};
chromalog = callPackage ./chromalog {};
txzmq = callPackage ./txzmq {};
joinmarketbase = joinmarketPkg ./jmbase;
joinmarketclient = joinmarketPkg ./jmclient;
joinmarketbitcoin = joinmarketPkg ./jmbitcoin;
joinmarketdaemon = joinmarketPkg ./jmdaemon;
pyln-client = clightningPkg ./pyln-client;
pyln-proto = clightningPkg ./pyln-proto;
pylightning = clightningPkg ./pylightning;
}

View File

@ -0,0 +1,17 @@
{ buildPythonPackage, clightning, pyln-client }:
buildPythonPackage rec {
pname = "pylightning";
version = "0.8.0"; # defined in ${src}/contrib/pyln-client/pyln/client/__init__.py
inherit (clightning) src;
propagatedBuildInputs = [ pyln-client ];
postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
# The clightning source contains pyln-client 0.8.0
postPatch = ''
substituteInPlace requirements.txt --replace pyln-client==0.7.3 pyln-client==0.8.0
'';
}

View File

@ -0,0 +1,12 @@
{ buildPythonPackage, clightning, recommonmark }:
buildPythonPackage rec {
pname = "pyln-client";
version = "0.8.0"; # defined in ${src}/contrib/pyln-client/pyln/client/__init__.py
inherit (clightning) src;
propagatedBuildInputs = [ recommonmark ];
postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
}

View File

@ -0,0 +1,31 @@
{ buildPythonPackage, clightning
, bitstring
, cryptography
, coincurve
, base58
, mypy
}:
buildPythonPackage rec {
pname = "pyln-proto";
version = "0.8.4"; # defined in ${src}/contrib/pyln-proto/setup.py
inherit (clightning) src;
propagatedBuildInputs = [
bitstring
cryptography
coincurve
base58
mypy
];
postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}";
postPatch = ''
substituteInPlace requirements.txt \
--replace base58==1.0.2 base58==2.0.1 \
--replace bitstring==3.1.6 bitstring==3.1.5 \
--replace cryptography==2.8 cryptography==3.1
'';
}

View File

@ -0,0 +1,28 @@
{ lib
, buildPythonPackage
, fetchPypi
, twisted
, pyzmq
}:
buildPythonPackage rec {
pname = "txzmq";
version = "0.8.2";
src = fetchPypi {
pname = "txZMQ";
inherit version;
sha256 = "07a9a480e58d4d732eef9efd7e264f2348cbf27ee82b338ec818a8504006e1c0";
};
propagatedBuildInputs = [
twisted
pyzmq
];
meta = with lib; {
description = "Twisted bindings for ZeroMQ";
homepage = https://github.com/smira/txZMQ;
license = licenses.gpl2;
};
}

View File

@ -3,7 +3,7 @@ testArgs:
let let
pkgs = import <nixpkgs> { config = {}; overlays = []; }; pkgs = import <nixpkgs> { config = {}; overlays = []; };
test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") testArgs; test = (import "${pkgs.path}/nixos/tests/make-test-python.nix") (testArgs pkgs);
fixedTest = { system ? builtins.currentSystem, ... }@args: fixedTest = { system ? builtins.currentSystem, ... }@args:
let let

View File

@ -1,7 +1,7 @@
scenario: testConfig: scenario: testConfig:
{ {
vm = import ./make-test-vm.nix { vm = import ./make-test-vm.nix (pkgs: {
name = "nix-bitcoin-${scenario}"; name = "nix-bitcoin-${scenario}";
machine = { machine = {
@ -16,7 +16,7 @@ scenario: testConfig:
data = cfg.test.data; data = cfg.test.data;
tests = cfg.tests; tests = cfg.tests;
}; };
dataFile = builtins.toFile "test-data" (builtins.toJSON data); dataFile = pkgs.writeText "test-data" (builtins.toJSON data);
initData = '' initData = ''
import json import json
@ -37,7 +37,7 @@ scenario: testConfig:
run_tests() run_tests()
'' ''
]; ];
}; });
container = { container = {
# The container name has a 11 char length limit # The container name has a 11 char length limit

View File

@ -21,7 +21,11 @@ let testEnv = rec {
} }
]; ];
config = { options.test.features = {
clightningPlugins = mkEnableOption "all clightning plugins";
};
config = mkMerge [{
tests.bitcoind = cfg.bitcoind.enable; tests.bitcoind = cfg.bitcoind.enable;
services.bitcoind = { services.bitcoind = {
enable = true; enable = true;
@ -31,6 +35,11 @@ let testEnv = rec {
tests.clightning = cfg.clightning.enable; tests.clightning = cfg.clightning.enable;
# When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s. # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s.
services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns"; services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns";
test.data.clightning-plugins = let
plugins = config.services.clightning.plugins;
enabled = builtins.filter (plugin: plugins.${plugin}.enable) (builtins.attrNames plugins);
pluginPkgs = config.nix-bitcoin.pkgs.clightning-plugins;
in map (plugin: pluginPkgs.${plugin}.path) enabled;
tests.spark-wallet = cfg.spark-wallet.enable; tests.spark-wallet = cfg.spark-wallet.enable;
@ -67,7 +76,28 @@ let testEnv = rec {
systemd.services.generate-secrets.postStart = mkIfTest "security" '' systemd.services.generate-secrets.postStart = mkIfTest "security" ''
install -o nobody -g nogroup -m777 <(:) /secrets/dummy install -o nobody -g nogroup -m777 <(:) /secrets/dummy
''; '';
}; }
(mkIf config.test.features.clightningPlugins {
services.clightning.plugins = {
helpme.enable = true;
monitor.enable = true;
prometheus.enable = true;
rebalance.enable = true;
summary.enable = true;
zmq = let tcpEndpoint = "tcp://127.0.0.1:5501"; in {
enable = true;
channel-opened = tcpEndpoint;
connect = tcpEndpoint;
disconnect = tcpEndpoint;
invoice-payment = tcpEndpoint;
warning = tcpEndpoint;
forward-event = tcpEndpoint;
sendpay-success = tcpEndpoint;
sendpay-failure = tcpEndpoint;
};
};
})
];
}; };
scenarios = { scenarios = {
@ -80,6 +110,7 @@ let testEnv = rec {
tests.security = true; tests.security = true;
services.clightning.enable = true; services.clightning.enable = true;
test.features.clightningPlugins = true;
services.spark-wallet.enable = true; services.spark-wallet.enable = true;
services.lightning-charge.enable = true; services.lightning-charge.enable = true;
services.nanopos.enable = true; services.nanopos.enable = true;
@ -120,6 +151,7 @@ let testEnv = rec {
regtest = { regtest = {
imports = [ scenarios.regtestBase ]; imports = [ scenarios.regtestBase ];
services.clightning.enable = true; services.clightning.enable = true;
test.features.clightningPlugins = true;
services.spark-wallet.enable = true; services.spark-wallet.enable = true;
services.lnd.enable = true; services.lnd.enable = true;
services.lightning-loop.enable = true; services.lightning-loop.enable = true;

View File

@ -1,4 +1,5 @@
from collections import OrderedDict from collections import OrderedDict
import json
def succeed(*cmds): def succeed(*cmds):
@ -138,6 +139,20 @@ def _():
def _(): def _():
assert_running("clightning") assert_running("clightning")
assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"') assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
if test_data["clightning-plugins"]:
plugin_list = succeed("lightning-cli plugin list")
plugins = json.loads(plugin_list)["plugins"]
active = set(plugin["name"] for plugin in plugins if plugin["active"])
failed = set(test_data["clightning-plugins"]).difference(active)
if failed:
raise Exception(
f"The following clightning plugins are inactive:\n{failed}.\n\n"
f"Output of 'lightning-cli plugin list':\n{plugin_list}"
)
else:
log.log("Active clightning plugins:")
for p in test_data["clightning-plugins"]:
log.log(os.path.basename(p))
@test("lnd") @test("lnd")