Merge #245: Fix tests without secure-node

bfed10b2fa run-tests: add command 'all' (Erik Arvstedt)
0a6b9beda5 run-tests: simplify setting default scenario (Erik Arvstedt)
1a32292e07 test: speed up clightning startup when offline (Erik Arvstedt)
7d1797cec7 clightning: add option 'extraConfig' (Erik Arvstedt)
e0117d56d1 spark-wallet: fix always-on onion-chef setting (Erik Arvstedt)
480d0d3959 liquid: fix bitcoin rpc settings (Erik Arvstedt)
c07e767889 test: add python test requirements (Erik Arvstedt)
9aa19c3fdd extract operator module (Erik Arvstedt)
2dd1a741f7 modules: group imports (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK bfed10b2fa

Tree-SHA512: caa6a38f9c9ad583fee8b4a705f5a94f037354eb49c2502c7b0aae620981df5c6630b1c862ead7fff2d9a47d0d15d3ce819567f63570e4d68ed02db7fe5f19a6

modules/bitcoind.nix

@@ -380,6 +380,7 @@ in {
     };
     users.groups.${cfg.group} = {};
     users.groups.bitcoinrpc = {};
+    nix-bitcoin.operator.groups = [ cfg.group ];
 
     nix-bitcoin.secrets.bitcoin-rpcpassword-privileged.user = "bitcoin";
     nix-bitcoin.secrets.bitcoin-rpcpassword-public = {

modules/clightning.nix

@@ -15,6 +15,7 @@ let
     ${optionalString (cfg.bitcoin-rpcconnect != null) "bitcoin-rpcconnect=${cfg.bitcoin-rpcconnect}"}
     bitcoin-rpcuser=${config.services.bitcoind.rpc.users.public.name}
     rpc-file-mode=0660
+    ${cfg.extraConfig}
   '';
 in {
   options.services.clightning = {
@@ -70,6 +71,11 @@ in {
       default = "/var/lib/clightning";
       description = "The data directory for clightning.";
     };
+    extraConfig = mkOption {
+      type = types.lines;
+      default = "";
+      description = "Additional lines appended to the config file.";
+    };
     user = mkOption {
       type = types.str;
       default = "clightning";
@@ -99,6 +105,7 @@ in {
         extraGroups = [ "bitcoinrpc" ];
     };
     users.groups.${cfg.group} = {};
+    nix-bitcoin.operator.groups = [ cfg.group ];
 
     systemd.tmpfiles.rules = [
       "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"

modules/hardware-wallets.nix

@@ -48,6 +48,7 @@ in {
         usbutils
       ];
       users.groups."${cfg.group}" = {};
+      nix-bitcoin.operator.groups = [ cfg.group ];
     })
     (mkIf cfg.ledger {
 

modules/joinmarket.nix

@@ -125,6 +125,10 @@ in {
         home = cfg.dataDir;
     };
     users.groups.${cfg.group} = {};
+    nix-bitcoin.operator = {
+      groups = [ cfg.group ];
+      sudoUsers = [ cfg.group ];
+    };
 
     systemd.tmpfiles.rules = [
       "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"

modules/liquid.nix

@@ -30,7 +30,9 @@ let
     ${lib.concatMapStrings (rpcallowip: "rpcallowip=${rpcallowip}\n") cfg.rpcallowip}
     ${optionalString (cfg.rpcuser != null) "rpcuser=${cfg.rpcuser}"}
     ${optionalString (cfg.rpcpassword != null) "rpcpassword=${cfg.rpcpassword}"}
-    ${optionalString (cfg.mainchainrpchost != null) "mainchainrpchost=${cfg.mainchainrpchost}"}
+    mainchainrpchost=${builtins.elemAt config.services.bitcoind.rpcbind 0}
+    mainchainrpcport=${toString config.services.bitcoind.rpc.port}
+    mainchainrpcuser=${config.services.bitcoind.rpc.users.public.name}
 
     # Extra config options (from liquidd nixos service)
     ${cfg.extraConfig}
@@ -146,15 +148,6 @@ in {
           default = null;
           description = "Password for JSON-RPC connections";
       };
-      mainchainrpchost = mkOption {
-        type = types.nullOr types.str;
-        default = null;
-        description = ''
-          The address which the daemon will try to connect to the trusted
-          mainchain daemon to validate peg-ins.
-        '';
-      };
-
       testnet = mkOption {
         type = types.bool;
         default = false;
@@ -263,12 +256,15 @@ in {
           else nix-bitcoin-services.allowAnyIP
         );
     };
+
     users.users.${cfg.user} = {
       group = cfg.group;
       extraGroups = [ "bitcoinrpc" ];
       description = "Liquid sidechain user";
     };
     users.groups.${cfg.group} = {};
+    nix-bitcoin.operator.groups = [ cfg.group ];
+
     nix-bitcoin.secrets.liquid-rpcpassword.user = "liquid";
   };
 }

modules/lnd.nix

@@ -259,6 +259,7 @@ in {
           else nix-bitcoin-services.allowAnyIP
         ) // nix-bitcoin-services.allowAnyProtocol;  # For ZMQ
     };
+
     users.users.lnd = {
       description = "LND User";
       group = "lnd";
@@ -266,6 +267,11 @@ in {
       home = cfg.dataDir; # lnd creates .lnd dir in HOME
     };
     users.groups.lnd = {};
+    nix-bitcoin.operator = {
+      groups = [ "lnd" ];
+      sudoUsers = [ "lnd" ];
+    };
+
     nix-bitcoin.secrets = {
       lnd-wallet-password.user = "lnd";
       lnd-key.user = "lnd";

modules/modules.nix

@@ -2,24 +2,30 @@
 
 {
   imports = [
+    # Core modules
+    ./secrets/secrets.nix
+    ./operator.nix
+
+    # Main features
     ./bitcoind.nix
     ./clightning.nix
     ./lightning-charge.nix
     ./nanopos.nix
-    ./liquid.nix
     ./spark-wallet.nix
-    ./electrs.nix
-    ./onion-chef.nix
-    ./recurring-donations.nix
-    ./hardware-wallets.nix
     ./lnd.nix
     ./lightning-loop.nix
-    ./secrets/secrets.nix
-    ./netns-isolation.nix
-    ./security.nix
-    ./backups.nix
     ./btcpayserver.nix
+    ./electrs.nix
+    ./liquid.nix
     ./joinmarket.nix
+    ./hardware-wallets.nix
+    ./recurring-donations.nix
+
+    # Support features
+    ./security.nix
+    ./netns-isolation.nix
+    ./backups.nix
+    ./onion-chef.nix
   ];
 
   disabledModules = [ "services/networking/bitcoind.nix" ];

modules/netns-isolation.nix

@@ -82,6 +82,7 @@ in {
         User that is allowed to execute commands in the service network namespaces.
         The user's group is also authorized.
       '';
+      default = config.nix-bitcoin.operator.name;
     };
 
     netns = mkOption {
@@ -294,7 +295,6 @@ in {
       rpcallowip = [
         "127.0.0.1"
       ] ++ map (n: "${netns.${n}.address}") netns.liquidd.availableNetns;
-      mainchainrpchost = netns.bitcoind.address;
       cliExec = mkCliExec "liquidd";
     };
 

modules/nodeinfo.nix

@@ -3,7 +3,7 @@
 with lib;
 
 let
-  operatorName = config.nix-bitcoin.operatorName;
+  operatorName = config.nix-bitcoin.operator.name;
   script = pkgs.writeScriptBin "nodeinfo" ''
     set -eo pipefail
 

modules/operator.nix

@@ -0,0 +1,47 @@
+# Define an operator user for convenient interactive access to nix-bitcoin
+# features and services.
+#
+# When using nix-bitcoin as part of a larger system config, set
+# `nix-bitcoin.operator.name` to your main user name.
+
+{ config, lib, pkgs, options, ... }:
+
+with lib;
+let
+  cfg = config.nix-bitcoin.operator;
+in {
+  options.nix-bitcoin.operator = {
+    enable = mkEnableOption "operator user";
+    name = mkOption {
+      type = types.str;
+      default = "operator";
+      description = "User name.";
+    };
+    groups = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = "Extra groups.";
+    };
+    sudoUsers = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = "Users as which the operator is allowed to run commands.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.users.${cfg.name} = {
+      isNormalUser = true;
+      extraGroups = [
+        "systemd-journal"
+        "proc" # Enable full /proc access and systemd-status
+      ] ++ cfg.groups;
+    };
+
+    security.sudo.extraConfig = mkIf (cfg.sudoUsers != []) (let
+      users = builtins.concatStringsSep "," cfg.sudoUsers;
+    in ''
+      ${cfg.name} ALL=(${users}) NOPASSWD: ALL
+    '');
+  };
+}

modules/presets/secure-node.nix

@@ -5,7 +5,7 @@ with lib;
 let
   cfg = config.services;
 
-  operatorName = config.nix-bitcoin.operatorName;
+  operatorName = config.nix-bitcoin.operator.name;
 
   mkHiddenService = map: {
     map = [ map ];
@@ -29,11 +29,6 @@ in {
       default = 9735;
       description = "Port on which to listen for tor client connections.";
     };
-    nix-bitcoin.operatorName = mkOption {
-      type = types.str;
-      default = "operator";
-      description = "Less-privileged user's name.";
-    };
   };
 
   config =  {
@@ -107,10 +102,6 @@ in {
     services.liquidd = {
       rpcuser = "liquidrpc";
       prune = 1000;
-      extraConfig = ''
-        mainchainrpcuser=${config.services.bitcoind.rpc.users.public.name}
-        mainchainrpcport=8332
-      '';
       validatepegin = true;
       listen = true;
       proxy = cfg.tor.client.socksListenAddress;
@@ -159,35 +150,15 @@ in {
       qrencode
     ];
 
-    # Create operator user which can access the node's services
+    services.onion-chef = {
+      enable = true;
+      access.${operatorName} = [ "bitcoind" "clightning" "nginx" "liquidd" "spark-wallet" "electrs" "btcpayserver" "sshd" ];
+    };
+
+    nix-bitcoin.operator.enable = true;
     users.users.${operatorName} = {
-      isNormalUser = true;
-      extraGroups = [
-          "systemd-journal"
-          "proc" # Enable full /proc access and systemd-status
-          cfg.bitcoind.group
-        ]
-        ++ (optionals cfg.clightning.enable [ "clightning" ])
-        ++ (optionals cfg.lnd.enable [ "lnd" ])
-        ++ (optionals cfg.liquidd.enable [ cfg.liquidd.group ])
-        ++ (optionals (cfg.hardware-wallets.ledger || cfg.hardware-wallets.trezor)
-            [ cfg.hardware-wallets.group ])
-        ++ (optionals cfg.joinmarket.enable [ cfg.joinmarket.group ]);
       openssh.authorizedKeys.keys = config.users.users.root.openssh.authorizedKeys.keys;
     };
-    nix-bitcoin.netns-isolation.allowedUser = operatorName;
-    # Give operator access to onion hostnames
-    services.onion-chef.enable = true;
-    services.onion-chef.access.${operatorName} = [ "bitcoind" "clightning" "nginx" "liquidd" "spark-wallet" "electrs" "btcpayserver" "sshd" ];
-
-    security.sudo.configFile =
-     (optionalString cfg.lnd.enable ''
-       ${operatorName}    ALL=(lnd) NOPASSWD: ALL
-     '') +
-     (optionalString cfg.joinmarket.enable ''
-       ${operatorName}    ALL=(${cfg.joinmarket.user}) NOPASSWD: ALL
-     '');
-
     # Enable nixops ssh for operator (`nixops ssh operator@mynode`) on nixops-vbox deployments
     systemd.services.get-vbox-nixops-client-key =
       mkIf (builtins.elem ".vbox-nixops-client-key" config.services.openssh.authorizedKeysFiles) {

modules/spark-wallet.nix

@@ -89,7 +89,7 @@ in {
         User = "spark-wallet";
         Restart = "on-failure";
         RestartSec = "10s";
-        ReadWritePaths = "/var/lib/onion-chef";
+        ReadWritePaths = mkIf cfg.onion-service "/var/lib/onion-chef";
       } // (if cfg.enforceTor
             then nix-bitcoin-services.allowTor
             else nix-bitcoin-services.allowAnyIP)

test/run-tests.sh

@@ -16,8 +16,8 @@
 #     Example:
 #     ./run-tests.sh -s electrs
 #
-#   Run test and link results to avoid garbage collection
+#   Run test(s) and link results to avoid garbage collection
-#   ./run-tests.sh [--scenario <scenario>] --out-link-prefix /tmp/nix-bitcoin-test build
+#   ./run-tests.sh [--scenario <scenario>] --out-link-prefix /tmp/nix-bitcoin-test
 #
 #   Pass extra args to nix-build
 #   ./run-tests.sh build --builders 'ssh://mybuildhost - - 15'
@@ -168,23 +168,34 @@ vmTestNixExpr() {
 EOF
 }
 
+# A basic subset of tests to keep the total runtime within
+# manageable bounds (<3 min on desktop systems).
+# These are also run on the CI server.
+basic() {
+    scenario=default buildTest "$@"
+    scenario=netns buildTest "$@"
+    scenario=full evalTest "$@"
+}
+
+all() {
+    scenario=default buildTest "$@"
+    scenario=netns buildTest "$@"
+    scenario=full buildTest "$@"
+}
+
 build() {
     if [[ $scenario ]]; then
         buildTest "$@"
     else
-        scenario=default buildTest "$@"
+        basic "$@"
-        scenario=netns buildTest "$@"
-        scenario=full evalTest "$@"
     fi
 }
 
-# Set default scenario for all actions other than 'build'
-if [[ $1 && $1 != build ]]; then
-    : ${scenario:=default}
-fi
-
 command="${1:-build}"
 shift || true
+if [[ $command != build  ]]; then
+    : ${scenario:=default}
+fi
 if [[ $command == eval ]]; then
     command=evalTest
 fi

test/tests.nix

@@ -13,6 +13,12 @@ let testEnv = rec {
       ./lib/test-lib.nix
       ../modules/modules.nix
       ../modules/secrets/generate-secrets.nix
+      {
+        # Features required by the Python test suite
+        nix-bitcoin.secretsDir = "/secrets";
+        nix-bitcoin.operator.enable = true;
+        environment.systemPackages = with pkgs; [ jq ];
+      }
     ];
 
     config = {
@@ -23,6 +29,8 @@ let testEnv = rec {
       };
 
       tests.clightning = cfg.clightning.enable;
+      # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s.
+      services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns";
 
       tests.spark-wallet = cfg.spark-wallet.enable;