add standalone clightning-rest service

- Rename `services.rtl.cl-rest` to `services.clightning-rest`.
  `clightning-rest` is generally useful for connecting external REST clients
  to clightning.

- Add a dedicated network namespace in netns-isolation.

- Add nodeinfo entry.

- Add datadir (which contains REST auth data) to backups.

README.md

@@ -73,6 +73,7 @@ NixOS modules ([src](modules/modules.nix))
     * [rebalance](https://github.com/lightningd/plugins/tree/master/rebalance): keeps your channels balanced
     * [summary](https://github.com/lightningd/plugins/tree/master/summary): print a nice summary of the node status
     * [zmq](https://github.com/lightningd/plugins/tree/master/zmq): publishes notifications via ZeroMQ to configured endpoints
+  * [clightning-rest](https://github.com/Ride-The-Lightning/c-lightning-REST): REST server for clightning
   * [lnd](https://github.com/lightningnetwork/lnd) with support for announcing an onion service and [static channel backups](https://github.com/lightningnetwork/lnd/blob/master/docs/recovery.md)
     * [Lightning Loop](https://github.com/lightninglabs/loop)
     * [Lightning Pool](https://github.com/lightninglabs/pool)

modules/backups.nix

@@ -62,6 +62,7 @@ let
     ''}
     ${config.services.bitcoind.dataDir}
     ${config.services.clightning.dataDir}
+    ${config.services.clightning-rest.dataDir}
     ${config.services.lnd.dataDir}
     ${optionalString (!cfg.with-bulk-data) ''
       - ${config.services.liquidd.dataDir}/*/blocks

modules/clightning-rest.nix

@@ -0,0 +1,104 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  options.services.clightning-rest = {
+    enable = mkEnableOption "lightning-rest";
+    port = mkOption {
+      type = types.port;
+      default = 3001;
+      description = "REST server port.";
+    };
+    docPort = mkOption {
+      type = types.port;
+      default = 4001;
+      description = "Swagger API documentation server port.";
+    };
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/clightning-rest";
+      description = "The data directory for clightning-rest.";
+    };
+    extraConfig = mkOption {
+      type = types.attrs;
+      default = {};
+      example = {
+        DOMAIN = "mynode.org";
+      };
+      description = ''
+        Extra config options.
+        See: https://github.com/Ride-The-Lightning/c-lightning-REST#option-1-via-config-file-cl-rest-configjson
+      '';
+    };
+    # Used by ./rtl.nix
+    group = mkOption {
+      readOnly = true;
+      default = clightning.group;
+      description = "The group under which clightning-rest is run.";
+    };
+    # Rest server address.
+    # Not configurable. The server always listens on all interfaces:
+    # https://github.com/Ride-The-Lightning/c-lightning-REST/issues/84
+    # Required by netns-isolation.
+    address = mkOption {
+      internal = true;
+      default = "0.0.0.0";
+    };
+    tor.enforce = nbLib.tor.enforce;
+  };
+
+  cfg = config.services.clightning-rest;
+  nbLib = config.nix-bitcoin.lib;
+  nbPkgs = config.nix-bitcoin.pkgs;
+
+  inherit (config.services)
+    bitcoind
+    clightning;
+
+  configFile = builtins.toFile "clightning-rest-config" (builtins.toJSON ({
+    PORT = cfg.port;
+    DOCPORT = cfg.docPort;
+    LNRPCPATH = "${clightning.dataDir}/${bitcoind.makeNetworkName "bitcoin" "regtest"}/lightning-rpc";
+    EXECMODE = "production";
+    PROTOCOL = "https";
+    RPCCOMMANDS = ["*"];
+  } // cfg.extraConfig));
+in {
+  inherit options;
+
+  config = mkIf cfg.enable {
+    services.clightning.enable = true;
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' 0770 ${clightning.user} ${cfg.group} - -"
+    ];
+
+    systemd.services.clightning-rest = mkIf cfg.enable {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "clightning.service" ];
+      after = [ "clightning.service" ];
+      path = [ pkgs.openssl ];
+      environment.CL_REST_STATE_DIR = cfg.dataDir;
+      preStart = ''
+        ln -sfn ${configFile} cl-rest-config.json
+      '';
+      postStart = ''
+        while [[ ! -e '${cfg.dataDir}/certs/access.macaroon' ]]; do
+          sleep 0.1
+        done
+      '';
+      serviceConfig = nbLib.defaultHardening // {
+        # clightning-rest reads the config file from the working directory
+        WorkingDirectory = cfg.dataDir;
+        ExecStart = "${nbPkgs.clightning-rest}/bin/cl-rest";
+        # Show "clightning-rest" instead of "node" in the journal
+        SyslogIdentifier = "clightning-rest";
+        User = clightning.user;
+        Restart = "on-failure";
+        RestartSec = "10s";
+        ReadWritePaths = cfg.dataDir;
+      } // nbLib.allowedIPAddresses cfg.tor.enforce
+        // nbLib.nodejs;
+    };
+  };
+}

modules/modules.nix

@@ -12,6 +12,7 @@
     ./bitcoind.nix
     ./clightning.nix
     ./clightning-plugins
+    ./clightning-rest.nix
     ./spark-wallet.nix
     ./lnd.nix
     ./lnd-rest-onion-service.nix # Requires onion-addresses.nix

modules/netns-isolation.nix

@@ -283,9 +283,13 @@ in {
       };
       rtl = {
         id = 29;
-        connections = []
+        connections =
-                      ++ optional (config.services.rtl.nodes.lnd) "lnd"
+          optional config.services.rtl.nodes.lnd "lnd" ++
-                      ++ optional config.services.rtl.loop "lightning-loop";
+          optional config.services.rtl.loop "lightning-loop" ++
+          optional config.services.rtl.nodes.clightning "clightning-rest";
+      };
+      clightning-rest = {
+        id = 30;
       };
     };
 
@@ -341,11 +345,8 @@ in {
     services.lightning-pool.rpcAddress = netns.lightning-pool.address;
 
     services.rtl.address = netns.rtl.address;
-    systemd.services.cl-rest = mkIf config.services.rtl.cl-rest.enable {
+
-      serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-rtl";
+    services.clightning-rest.address = netns.clightning-rest.address;
-      requires = [ "netns-rtl.service" ] ;
-      after = [ "netns-rtl.service" ];
-    };
   }
   ]);
 }

modules/nodeinfo.nix

@@ -27,6 +27,7 @@ let
     lnd = mkInfo ''
       info["nodeid"] = shell("lncli getinfo | jq -r '.identity_pubkey'")
     '';
+    clightning-rest = mkInfo "";
     electrs = mkInfo "";
     spark-wallet = mkInfo "";
     btcpayserver = mkInfo "";

modules/obsolete-options.nix

@@ -1,4 +1,4 @@
-{ lib, ... }:
+{ lib, config, ... }:
 
 with lib;
 let
@@ -31,6 +31,8 @@ in {
     (mkRenamedOptionModule [ "services" "btcpayserver" "bind" ] [ "services" "btcpayserver" "address" ])
     (mkRenamedOptionModule [ "services" "liquidd" "bind" ] [ "services" "liquidd" "address" ])
     (mkRenamedOptionModule [ "services" "liquidd" "rpcbind" ] [ "services" "liquidd" "rpc" "address" ])
+    # 0.0.70
+    (mkRenamedOptionModule [ "services" "rtl" "cl-rest" ] [ "services" "clightning-rest" ])
 
     (mkRenamedOptionModule [ "nix-bitcoin" "setup-secrets" ] [ "nix-bitcoin" "setupSecrets" ])
 
@@ -59,4 +61,22 @@ in {
     "rtl"
     "electrs"
   ]);
+
+  config = {
+    # Migrate old clightning-rest datadir from nix-bitcoin versions < 0.0.70
+    systemd.services.clightning-rest-migrate-datadir = let
+      inherit (config.services) clightning-rest clightning;
+    in mkIf config.services.clightning-rest.enable {
+      requiredBy = [ "clightning-rest.service" ];
+      before = [ "clightning-rest.service" ];
+      script = ''
+        if [[ -e /var/lib/cl-rest/certs ]]; then
+          mv /var/lib/cl-rest/* '${clightning-rest.dataDir}'
+          chown -R ${clightning.user}: '${clightning-rest.dataDir}'
+          rm -r /var/lib/cl-rest
+        fi
+      '';
+      serviceConfig.Type = "oneshot";
+    };
+  };
 }

modules/presets/enable-tor.nix

@@ -38,6 +38,7 @@ in {
     rtl = defaultEnforceTor;
     joinmarket = defaultEnforceTor;
     joinmarket-ob-watcher = defaultEnforceTor;
+    clightning-rest = defaultEnforceTor;
   };
 
   # Add onion services for incoming connections

modules/rtl.nix

@@ -70,36 +70,6 @@ let
       default = cfg.user;
       description = "The group as which to run RTL.";
     };
-    cl-rest = {
-      enable = mkOption {
-        readOnly = true;
-        type = types.bool;
-        default = cfg.nodes.clightning;
-        description = ''
-          Enable c-lightning-REST server. This service is required for
-          clightning support and is automatically enabled.
-        '';
-      };
-      address = mkOption {
-        readOnly = true;
-        default = "0.0.0.0";
-        description = ''
-          Rest server address.
-          Not configurable. The server always listens on all interfaces:
-          https://github.com/Ride-The-Lightning/c-lightning-REST/issues/84
-        '';
-      };
-      port = mkOption {
-        type = types.port;
-        default = 3001;
-        description = "REST server port.";
-      };
-      docPort = mkOption {
-        type = types.port;
-        default = 4001;
-        description = "Swagger API documentation server port.";
-      };
-    };
     tor.enforce = nbLib.tor.enforce;
   };
 
@@ -119,7 +89,7 @@ let
          }
         "macaroonPath": "${if isLnd
                            then "${cfg.dataDir}/macaroons"
-                           else "${cl-rest.dataDir}/certs"
+                           else "${clightning-rest.dataDir}/certs"
                           }"
       },
       "Settings": {
@@ -140,7 +110,7 @@ let
         "lnServerUrl": "https://${
           if isLnd
           then nbLib.addressWithPort lnd.restAddress lnd.restPort
-          else nbLib.addressWithPort cfg.cl-rest.address cfg.cl-rest.port
+          else nbLib.addressWithPort clightning-rest.address clightning-rest.port
         }"
       }
     }
@@ -165,25 +135,10 @@ let
     }
   '';
 
-  cl-rest = {
-    configFile = builtins.toFile "config" ''
-      {
-	      "PORT": ${toString cfg.cl-rest.port},
-        "DOCPORT": ${toString cfg.cl-rest.docPort},
-        "LNRPCPATH": "${clightning.dataDir}/${bitcoind.makeNetworkName "bitcoin" "regtest"}/lightning-rpc",
-	      "PROTOCOL": "https",
-	      "EXECMODE": "production",
-	      "RPCCOMMANDS": ["*"]
-      }
-    '';
-    # serviceConfig.StateDirectory
-    dataDir = "/var/lib/cl-rest";
-  };
-
   inherit (config.services)
     bitcoind
     lnd
-    clightning
+    clightning-rest
     lightning-loop;
 in {
   inherit options;
@@ -199,7 +154,7 @@ in {
 
     services.lnd.enable = mkIf cfg.nodes.lnd true;
     services.lightning-loop.enable = mkIf cfg.loop true;
-    services.clightning.enable = mkIf cfg.nodes.clightning true;
+    services.clightning-rest.enable = mkIf cfg.nodes.clightning true;
 
     systemd.tmpfiles.rules = [
       "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
@@ -209,7 +164,7 @@ in {
 
     systemd.services.rtl = rec {
       wantedBy = [ "multi-user.target" ];
-      requires = optional cfg.nodes.clightning "cl-rest.service" ++
+      requires = optional cfg.nodes.clightning "clightning-rest.service" ++
                  optional cfg.nodes.lnd "lnd.service";
       after = requires;
       environment.RTL_CONFIG_PATH = cfg.dataDir;
@@ -235,35 +190,12 @@ in {
         // nbLib.nodejs;
     };
 
-    systemd.services.cl-rest = mkIf cfg.cl-rest.enable {
-      wantedBy = [ "multi-user.target" ];
-      requires = [ "clightning.service" ];
-      after = [ "clightning.service" ];
-      path = [ pkgs.openssl ];
-      preStart = ''
-        ln -sfn ${cl-rest.configFile} cl-rest-config.json
-      '';
-      environment.CL_REST_STATE_DIR = cl-rest.dataDir;
-      serviceConfig = nbLib.defaultHardening // {
-        StateDirectory = "cl-rest";
-        # cl-rest reads the config file from the working directory
-        WorkingDirectory = cl-rest.dataDir;
-        ExecStart = "${nbPkgs.clightning-rest}/bin/cl-rest";
-        # Show "cl-rest" instead of "node" in the journal
-        SyslogIdentifier = "cl-rest";
-        User = cfg.user;
-        Restart = "on-failure";
-        RestartSec = "10s";
-      } // nbLib.allowLocalIPAddresses
-        // nbLib.nodejs;
-    };
-
     users.users.${cfg.user} = {
       isSystemUser = true;
       group = cfg.group;
       extraGroups =
-        # Enable clightning RPC access for cl-rest
+        # Reads cert and macaroon from the clightning-rest datadir
-        optional cfg.cl-rest.enable clightning.group ++
+        optional cfg.nodes.clightning clightning-rest.group ++
         optional cfg.loop lnd.group;
     };
     users.groups.${cfg.group} = {};

modules/versioning.nix

@@ -213,6 +213,17 @@ let
         See also: https://github.com/dgarage/NBXplorer/blob/master/docs/Postgres-Migration.md
       '';
     }
+    {
+      version = "0.0.70";
+      condition = config.services.clightning-rest.enable;
+      message = ''
+        The `cl-rest` service has been renamed to `clightning-rest`.
+        and is now available as a standalone service (`services.clightning-rest`).
+        Its data dir has moved to `${config.services.clightning-rest.dataDir}`,
+        and the service now runs under the clightning user and group.
+        The data dir migration happens automatically after deploying.
+      '';
+    }
   ];
 
   mkOnionServiceChange = service: {

test/tests.nix

@@ -59,6 +59,8 @@ let
       systemd.services.clightning.serviceConfig.TimeoutStopSec =
         mkIf config.services.clightning.plugins.clboss.enable "500ms";
 
+      tests.clightning-rest = cfg.clightning-rest.enable;
+
       tests.rtl = cfg.rtl.enable;
       services.rtl.nodes.lnd = mkDefault true;
       services.rtl.nodes.clightning = mkDefault true;
@@ -163,6 +165,7 @@ let
       test.features.clightningPlugins = true;
       services.rtl.enable = true;
       services.spark-wallet.enable = true;
+      services.clightning-rest.enable = true;
       services.lnd.enable = true;
       services.lnd.restOnionService.enable = true;
       services.lightning-loop.enable = true;
@@ -206,6 +209,7 @@ let
       imports = [ scenarios.regtestBase ];
       services.clightning.enable = true;
       test.features.clightningPlugins = true;
+      services.clightning-rest.enable = true;
       services.liquidd.enable = true;
       services.rtl.enable = true;
       services.spark-wallet.enable = true;

test/tests.py

@@ -211,9 +211,12 @@ def _():
     machine.wait_until_succeeds(
         log_has_string("rtl", "Server is up and running")
     )
-    assert_running("cl-rest")
+
+@test("clightning-rest")
+def _():
+    assert_running("clightning-rest")
     machine.wait_until_succeeds(
-        log_has_string("cl-rest", "cl-rest api server is ready and listening on port: 3001")
+        log_has_string("clightning-rest", "cl-rest api server is ready and listening on port: 3001")
     )
 
 @test("spark-wallet")