clightning: native database replication

Don't put `clightning.replication` options in
`examples/configuration.nix` until it is more "battle-tested."

test/clightning-replication.nix

@@ -0,0 +1,153 @@
+# You can run this test via `run-tests.sh -s clightningReplication`
+
+let
+  nixpkgs = (import ../pkgs/nixpkgs-pinned.nix).nixpkgs;
+in
+import "${nixpkgs}/nixos/tests/make-test-python.nix" ({ pkgs, ... }:
+with pkgs.lib;
+let
+  keyDir = "${nixpkgs}/nixos/tests/initrd-network-ssh";
+  keys = {
+    server = "${keyDir}/ssh_host_ed25519_key";
+    client = "${keyDir}/id_ed25519";
+    serverPub = readFile "${keys.server}.pub";
+    clientPub = readFile "${keys.client}.pub";
+  };
+
+  clientBaseConfig = {
+    imports = [ ../modules/modules.nix ];
+
+    nix-bitcoin.generateSecrets = true;
+
+    services.clightning = {
+      enable = true;
+      replication.enable = true;
+
+      # TODO-EXTERNAL:
+      # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s.
+      extraConfig = "disable-dns";
+    };
+  };
+in
+{
+  name = "clightning-replication";
+
+  nodes = let nodes = {
+    replicationLocal = {
+      imports = [ clientBaseConfig ];
+      services.clightning.replication.local.directory = "/var/backup/clightning";
+    };
+
+    replicationLocalEncrypted = {
+      imports = [ nodes.replicationLocal ];
+      services.clightning.replication.encrypt = true;
+    };
+
+    replicationRemote = {
+      imports = [ clientBaseConfig ];
+      nix-bitcoin.generateSecretsCmds.clightning-replication-ssh-key = mkForce ''
+        install -m 600 ${keys.client} clightning-replication-ssh-key
+      '';
+      programs.ssh.knownHosts."server".publicKey = keys.serverPub;
+      services.clightning.replication.sshfs.destination = "nb-replication@server:writable";
+    };
+
+    replicationRemoteEncrypted = {
+      imports = [ nodes.replicationRemote ];
+      services.clightning.replication.encrypt = true;
+    };
+
+    server = { ... }: {
+      environment.etc."ssh-host-key" = {
+        source = keys.server;
+        mode = "400";
+      };
+
+      services.openssh = {
+        enable = true;
+        extraConfig = ''
+          Match user nb-replication
+            ChrootDirectory /var/backup/nb-replication
+            AllowTcpForwarding no
+            AllowAgentForwarding no
+            ForceCommand internal-sftp
+            PasswordAuthentication no
+            X11Forwarding no
+        '';
+        hostKeys = mkForce [
+          {
+            path = "/etc/ssh-host-key";
+            type = "ed25519";
+          }
+        ];
+      };
+
+      users.users.nb-replication = {
+        isSystemUser = true;
+        group = "nb-replication";
+        shell = "${pkgs.coreutils}/bin/false";
+        openssh.authorizedKeys.keys = [ keys.clientPub ];
+      };
+      users.groups.nb-replication = {};
+
+      systemd.tmpfiles.rules = [
+        # Because this directory is chrooted by sshd, it must only be writable by user/group root
+        "d /var/backup/nb-replication 0755 root root - -"
+        "d /var/backup/nb-replication/writable 0700 nb-replication - - -"
+      ];
+    };
+  }; in nodes;
+
+  testScript =  { nodes, ... }: let
+    systems = builtins.concatStringsSep ", "
+      (mapAttrsToList (name: node: ''"${name}": "${node.config.system.build.toplevel}"'') nodes);
+  in ''
+    systems = { ${systems} }
+
+    def switch_to_system(system):
+        cmd = f"{systems[system]}/bin/switch-to-configuration test >&2"
+        client.succeed(cmd)
+
+    client = replicationLocal
+
+    if not "is_interactive" in vars():
+      client.start()
+      server.start()
+
+      with subtest("local replication"):
+          client.wait_for_unit("clightning.service")
+          client.succeed("runuser -u clightning -- ls /var/backup/clightning/lightningd.sqlite3")
+          # No other user should be able to read the backup directory
+          client.fail("runuser -u bitcoin -- ls /var/backup/clightning")
+
+      # If `switch_to_system` succeeds then all services, including clightning,
+      # have started successfully
+      switch_to_system("replicationLocalEncrypted")
+      with subtest("local replication encrypted"):
+          replica_db = "/var/cache/clightning-replication/plaintext/lightningd.sqlite3"
+          client.succeed(f"runuser -u clightning -- ls {replica_db}")
+          # No other user should be able to read the unencrypted files
+          client.fail(f"runuser -u bitcoin -- ls {replica_db}")
+          # A gocryptfs has been created
+          client.succeed("ls /var/backup/clightning/lightningd-db/gocryptfs.conf")
+
+      server.wait_for_unit("sshd.service")
+      switch_to_system("replicationRemote")
+      with subtest("remote replication"):
+          replica_db = "/var/cache/clightning-replication/sshfs/lightningd.sqlite3"
+          client.succeed(f"runuser -u clightning -- ls {replica_db}")
+          # No other user should be able to read the unencrypted files
+          client.fail(f"runuser -u bitcoin -- ls {replica_db}")
+          # A clighting db exists on the server
+          server.succeed("ls /var/backup/nb-replication/writable/lightningd.sqlite3")
+
+      switch_to_system("replicationRemoteEncrypted")
+      with subtest("remote replication encrypted"):
+          replica_db = "/var/cache/clightning-replication/plaintext/lightningd.sqlite3"
+          client.succeed(f"runuser -u clightning -- ls {replica_db}")
+          # No other user should be able to read the unencrypted files
+          client.fail(f"runuser -u bitcoin -- ls {replica_db}")
+          # A gocryptfs has been created on the server
+          server.succeed("ls /var/backup/nb-replication/writable/lightningd-db/gocryptfs.conf")
+  '';
+})

test/lib/make-test.nix

@@ -55,10 +55,29 @@ name: testConfig:
   container = {
     # The container name has a 11 char length limit
     containers.nb-test = { config, ... }: {
-      config = {
-        extra = config.config.test.container;
-        config = testConfig;
-      };
+      imports = [
+        {
+          config = {
+            extra = config.config.test.container;
+            config = testConfig;
+          };
+        }
+
+        # Enable FUSE inside the container when clightning replication
+        # is enabled.
+        # TODO-EXTERNAL: Remove this when
+        # https://github.com/systemd/systemd/issues/17607
+        # has been resolved. This will also improve security.
+        (
+          let
+            clightning = config.config.services.clightning;
+          in
+            lib.mkIf (clightning.enable && clightning.replication.enable) {
+              bindMounts."/dev/fuse" = { hostPath = "/dev/fuse"; };
+              allowedDevices = [ { node = "/dev/fuse"; modifier = "rw"; } ];
+            }
+        )
+      ];
     };
   };
 

test/run-tests.sh

@@ -295,10 +295,11 @@ basic() {
 # All tests that only consist of building a nix derivation.
 # Their output is cached in /nix/store.
 buildable() {
-    basic
+    basic "$@"
     scenario=full buildTest "$@"
     scenario=regtest buildTest "$@"
     scenario=hardened buildTest "$@"
+    scenario=clightningReplication buildTest "$@"
 }
 
 examples() {

test/tests.nix

@@ -49,6 +49,8 @@ let
       };
 
       tests.clightning = cfg.clightning.enable;
+      test.data.clightning-replication = cfg.clightning.replication.enable;
+
       # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s.
       services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns";
       test.data.clightning-plugins = let
@@ -186,6 +188,11 @@ let
       tests.security = true;
 
       services.clightning.enable = true;
+      services.clightning.replication = {
+        enable = true;
+        encrypt = true;
+        local.directory = "/var/backup/clightning";
+      };
       test.features.clightningPlugins = true;
       services.rtl.enable = true;
       services.spark-wallet.enable = true;
@@ -354,7 +361,12 @@ let
     };
   makeTest' = import ./lib/make-test.nix pkgs;
 
-  tests = builtins.mapAttrs makeTest allScenarios;
+  tests = builtins.mapAttrs makeTest allScenarios // {
+    clightningReplication.vm = import ./clightning-replication.nix {
+      inherit pkgs;
+      inherit (pkgs.stdenv) system;
+    };
+  };
 
   getTest = name: tests.${name} or (makeTest name {
     services.${name}.enable = true;

test/tests.py

@@ -153,6 +153,14 @@ def _():
             # This is a one-shot service, so this command only succeeds if the service succeeds
             succeed("systemctl start clightning-feeadjuster")
 
+    if test_data["clightning-replication"]:
+        replica_db = "/var/cache/clightning-replication/plaintext/lightningd.sqlite3"
+        succeed(f"runuser -u clightning -- ls {replica_db}")
+        # No other user should be able to read the unencrypted files
+        machine.fail(f"runuser -u bitcoin -- ls {replica_db}")
+        # A gocryptfs has been created
+        succeed("ls /var/backup/clightning/lightningd-db/gocryptfs.conf")
+
 @test("lnd")
 def _():
     assert_running("lnd")