Merge #226: Improve netns-isolation and tests

e5fb3f6a7f run-tests: document how to pass extra build args (Erik Arvstedt)
df790f6766 run-tests: allow linking test build results for all scenarios (Erik Arvstedt)
91697b1427 test: allow for testing all scenarios (Erik Arvstedt)
28236691aa test: rename scenarios/lib.py -> base.py (Erik Arvstedt)
80da0a41bc test: load complete test environment in debug mode (Erik Arvstedt)
9b4cd7bd1c test: simplify scenario handling (Erik Arvstedt)
0f56ea6ad1 test: include scenario in test name (Erik Arvstedt)
9237e5dc3d test: use pydoc docstring (Erik Arvstedt)
ed73627e02 netns-exec: minor style fixes (Erik Arvstedt)
91ebc2d517 netns-exec: simplify installation (Erik Arvstedt)
809e754851 netns: improve bridge setup (Erik Arvstedt)
b7450877a0 netns: rename bridge peer devices br-nb-veth* -> nb-veth-br* (Erik Arvstedt)
8bfb7bb2f8 netns: rename bridge br0 -> nb-br (Erik Arvstedt)
32e70a7516 netns: move webindex config for modules-only usage (Erik Arvstedt)
121301337b netns: add option 'allowedUser' for modules-only usage (Erik Arvstedt)
9715134f06 netns: don't repeat cli definitions (Erik Arvstedt)
e385c73256 netns: separate implementation and service configs (Erik Arvstedt)
d0b8d77de2 netns: remove conditionals for service settings (Erik Arvstedt)
0f0f6ddbb9 netns: add comment about undesirable algorithmic complexity (Erik Arvstedt)
a3ae8668e6 netns: use map instead of concatMap (Erik Arvstedt)
b7fc819be5 netns: consistent var naming (Erik Arvstedt)
5a81693ef3 netns: add range check for netns ids (Erik Arvstedt)
74f1610668 netns: clarify addressblock description (Erik Arvstedt)
4eb92df08c netns: remove redundant filter (Erik Arvstedt)
50de54aef1 netns: remove empty connections defs (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK e5fb3f6a7f
  nixbitcoin:
    ACK e5fb3f6a7f

Tree-SHA512: e2accf7b5ab5d4c4c07a8f9307409021809326648139424ff7ebaa7be3e628f21d5be8dafabe19b9659d09537a5b3976e2513bc287e79027376b5271006bc214

modules/bitcoind.nix

@@ -265,20 +265,16 @@ in {
       };
       cli = mkOption {
         type = types.package;
-        default = cfg.cli-nonetns-exec;
+        # Overriden on netns-isolation
+        default = cfg.cliBase;
         description = "Binary to connect with the bitcoind instance.";
       };
-      # Needed because bitcoin-cli commands executed through systemd already
-      # run inside nb-bitcoind, hence they don't need netns-exec prefixed.
-      cli-nonetns-exec = mkOption {
+      cliBase = mkOption {
         readOnly = true;
         type = types.package;
         default = pkgs.writeScriptBin "bitcoin-cli" ''
           exec ${cfg.package}/bin/bitcoin-cli -datadir='${cfg.dataDir}' "$@"
         '';
-        description = ''
-          Binary to connect with the bitcoind instance without netns-exec.
-        '';
       };
       enforceTor =  nix-bitcoin-services.enforceTor;
     };
@@ -315,7 +311,7 @@ in {
         fi
       '';
       postStart = ''
-        cd ${cfg.cli-nonetns-exec}/bin
+        cd ${cfg.cliBase}/bin
         # Poll until bitcoind accepts commands. This can take a long time.
         while ! ./bitcoin-cli getnetworkinfo &> /dev/null; do
           sleep 1
@@ -342,7 +338,7 @@ in {
       bindsTo = [ "bitcoind.service" ];
       after = [ "bitcoind.service" ];
       script = ''
-        cd ${cfg.cli-nonetns-exec}/bin
+        cd ${cfg.cliBase}/bin
         echo "Importing node banlist..."
         cat ${./banlist.cli.txt} | while read line; do
             if ! err=$(eval "$line" 2>&1) && [[ $err != *already\ banned* ]]; then

modules/lightning-loop.nix

@@ -30,10 +30,11 @@ in {
       default = pkgs.writeScriptBin "loop"
       # Switch user because lnd makes datadir contents readable by user only
       ''
-        exec sudo -u lnd ${cfg.package}/bin/loop "$@"
+        ${cfg.cliExec} sudo -u lnd ${cfg.package}/bin/loop "$@"
       '';
       description = "Binary to connect with the lnd instance.";
     };
+    inherit (nix-bitcoin-services) cliExec;
     enforceTor =  nix-bitcoin-services.enforceTor;
   };
 

modules/liquid.nix

@@ -210,17 +210,19 @@ in {
         '';
       };
       cli = mkOption {
+        readOnly = true;
         default = pkgs.writeScriptBin "elements-cli" ''
-          exec ${pkgs.nix-bitcoin.elementsd}/bin/elements-cli -datadir='${cfg.dataDir}' "$@"
+          ${cfg.cliExec} ${pkgs.nix-bitcoin.elementsd}/bin/elements-cli -datadir='${cfg.dataDir}' "$@"
         '';
         description = "Binary to connect with the liquidd instance.";
       };
-      swap-cli = mkOption {
+      swapCli = mkOption {
         default = pkgs.writeScriptBin "liquidswap-cli" ''
-          exec ${pkgs.nix-bitcoin.liquid-swap}/bin/liquidswap-cli -c '${cfg.dataDir}/elements.conf' "$@"
+          ${cfg.cliExec} ${pkgs.nix-bitcoin.liquid-swap}/bin/liquidswap-cli -c '${cfg.dataDir}/elements.conf' "$@"
         '';
         description = "Binary for managing liquid swaps.";
       };
+      inherit (nix-bitcoin-services) cliExec;
       enforceTor =  nix-bitcoin-services.enforceTor;
     };
   };
@@ -229,7 +231,7 @@ in {
     environment.systemPackages = [
       pkgs.nix-bitcoin.elementsd
       (hiPrio cfg.cli)
-      (hiPrio cfg.swap-cli)
+      (hiPrio cfg.swapCli)
     ];
 
     systemd.tmpfiles.rules = [

modules/lnd.nix

@@ -115,11 +115,12 @@ in {
       default = pkgs.writeScriptBin "lncli"
       # Switch user because lnd makes datadir contents readable by user only
       ''
-        exec sudo -u lnd ${cfg.package}/bin/lncli --tlscertpath ${secretsDir}/lnd-cert \
+        ${cfg.cliExec} sudo -u lnd ${cfg.package}/bin/lncli --tlscertpath ${secretsDir}/lnd-cert \
           --macaroonpath '${cfg.dataDir}/chain/bitcoin/mainnet/admin.macaroon' "$@"
       '';
       description = "Binary to connect with the lnd instance.";
     };
+    inherit (nix-bitcoin-services) cliExec;
     enforceTor =  nix-bitcoin-services.enforceTor;
   };
 

modules/netns-isolation.nix

@@ -8,7 +8,8 @@ let
   netns = builtins.mapAttrs (n: v: {
     inherit (v) id;
     address = "169.254.${toString cfg.addressblock}.${toString v.id}";
-    availableNetns = builtins.filter isEnabled availableNetns.${n};
+    availableNetns = availableNetns.${n};
+    netnsName = "nb-${n}";
   }) enabledServices;
 
   # Symmetric netns connection matrix
@@ -16,6 +17,12 @@ let
   #   availableNetns.bitcoind = [ "clighting" ];
   #   and
   #   availableNetns.clighting = [ "bitcoind" ];
+  #
+  # FIXME: Although negligible for our purposes, this calculation's runtime
+  # is in the order of (number of connections * number of services),
+  # because attrsets and lists are fully copied on each update with '//' or '++'.
+  # This can only be improved with an update in the nix language.
+  #
   availableNetns = let
     # base = { clightning = [ "bitcoind" ]; ... }
     base = builtins.mapAttrs (n: v:
@@ -36,6 +43,7 @@ let
 
   bridgeIp = "169.254.${toString cfg.addressblock}.10";
 
+  mkCliExec = service: "exec netns-exec ${netns.${service}.netnsName}";
 in {
   options.nix-bitcoin.netns-isolation = {
     enable = mkEnableOption "netns isolation";
@@ -44,7 +52,7 @@ in {
       type = types.ints.u8;
       default = "1";
       description = ''
-        Specify the N address block in 169.254.N.0/24.
+        The address block N in 169.254.N.0/24, used as the prefix for netns addresses.
       '';
     };
 
@@ -53,12 +61,11 @@ in {
       type = types.attrsOf (types.submodule {
         options = {
           id = mkOption {
-            # TODO: Exclude 10
             # TODO: Assert uniqueness
-            type = types.int;
+            type = types.ints.between 11 255;
             description = ''
-              id for the netns, that is used for the IP address host part and
-              naming the interfaces. Must be unique. Must not be 10.
+              id for the netns, used for the IP address host part and
+              for naming the interfaces. Must be unique. Must be greater than 10.
             '';
           };
           connections = mkOption {
@@ -68,21 +75,118 @@ in {
         };
       });
     };
+
+    allowedUser = mkOption {
+      type = types.str;
+      description = ''
+        User that is allowed to execute commands in the service network namespaces.
+        The user's group is also authorized.
+      '';
     };
 
-  config = mkIf cfg.enable {
-    # Prerequisites
-    networking.dhcpcd.denyInterfaces = [ "br0" "br-nb*" "nb-veth*" ];
+    netns = mkOption {
+      default = netns;
+      readOnly = true;
+      description = "Exposes netns parameters.";
+    };
+  };
+
+  config = mkIf cfg.enable (mkMerge [
+
+  # Base infrastructure
+  {
+    networking.dhcpcd.denyInterfaces = [ "nb-br" "nb-veth*" ];
     services.tor.client.socksListenAddress = "${bridgeIp}:9050";
-    networking.firewall.interfaces.br0.allowedTCPPorts = [ 9050 ];
+    networking.firewall.interfaces.nb-br.allowedTCPPorts = [ 9050 ];
     boot.kernel.sysctl."net.ipv4.ip_forward" = true;
+
     security.wrappers.netns-exec = {
-      source  = "${pkgs.nix-bitcoin.netns-exec}/netns-exec";
+      source = pkgs.nix-bitcoin.netns-exec;
       capabilities = "cap_sys_admin=ep";
-      owner = "${config.nix-bitcoin.operatorName}";
+      owner = cfg.allowedUser;
       permissions = "u+rx,g+rx,o-rwx";
     };
 
+    systemd.services = {
+      # Due to a NixOS bug we can't currently use option `networking.bridges` to
+      # setup the bridge while `networking.useDHCP` is enabled.
+      nb-netns-bridge = {
+        description = "nix-bitcoin netns bridge";
+        wantedBy = [ "network-setup.service" ];
+        partOf = [ "network-setup.service" ];
+        before = [ "network-setup.service" ];
+        after = [ "network-pre.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = "yes";
+        };
+        script = ''
+          ${ip} link add name nb-br type bridge
+          ${ip} link set nb-br up
+          ${ip} addr add ${bridgeIp}/24 brd + dev nb-br
+          ${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
+        '';
+        preStop = ''
+          ${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
+          ${ip} link del nb-br
+        '';
+      };
+    } //
+    (let
+      makeNetnsServices = n: v: let
+        veth = "nb-veth-${toString v.id}";
+        peer = "nb-veth-br-${toString v.id}";
+        inherit (v) netnsName;
+        ipNetns = "${ip} -n ${netnsName}";
+        netnsIptables = "${ip} netns exec ${netnsName} ${config.networking.firewall.package}/bin/iptables";
+      in {
+        "${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}";
+
+        "netns-${n}" = rec {
+          requires = [ "nb-netns-bridge.service" ];
+          after = [ "nb-netns-bridge.service" ];
+          bindsTo = [ "${n}.service" ];
+          requiredBy = bindsTo;
+          before = bindsTo;
+          script = ''
+            ${ip} netns add ${netnsName}
+            ${ipNetns} link set lo up
+            ${ip} link add ${veth} type veth peer name ${peer}
+            ${ip} link set ${veth} netns ${netnsName}
+            ${ipNetns} addr add ${v.address}/24 dev ${veth}
+            ${ip} link set ${peer} up
+            ${ipNetns} link set ${veth} up
+            ${ip} link set ${peer} master nb-br
+            ${ipNetns} route add default via ${bridgeIp}
+            ${netnsIptables} -w -P INPUT DROP
+            ${netnsIptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
+          '' + (optionalString (config.services.${n}.enforceTor or false)) ''
+            ${netnsIptables} -w -P OUTPUT DROP
+            ${netnsIptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
+          '' + concatMapStrings (otherNetns: let
+            other = netns.${otherNetns};
+          in ''
+            ${netnsIptables} -w -A INPUT -s ${other.address} -j ACCEPT
+            ${netnsIptables} -w -A OUTPUT -d ${other.address} -j ACCEPT
+          '') v.availableNetns;
+          preStop = ''
+            ${ip} netns delete ${netnsName}
+            ${ip} link del ${peer}
+          '';
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = "yes";
+            ExecStartPre = "-${ip} netns delete ${netnsName}";
+          };
+        };
+      };
+    in foldl (services: n:
+      services // (makeNetnsServices n netns.${n})
+    ) {} (builtins.attrNames netns));
+  }
+
+  # Service-specific config
+  {
     nix-bitcoin.netns-isolation.services = {
       bitcoind = {
         id = 12;
@@ -106,12 +210,10 @@ in {
       spark-wallet = {
         id = 17;
         # communicates with clightning over lightning-rpc socket
-        connections = [];
       };
       lightning-charge = {
         id = 18;
         # communicates with clightning over lightning-rpc socket
-        connections = [];
       };
       nanopos = {
         id = 19;
@@ -120,11 +222,9 @@ in {
       recurring-donations = {
         id = 20;
         # communicates with clightning over lightning-rpc socket
-        connections = [];
       };
       nginx = {
         id = 21;
-        connections = [];
       };
       lightning-loop = {
         id = 22;
@@ -132,81 +232,6 @@ in {
       };
     };
 
-    systemd.services = {
-      netns-bridge = {
-        description = "Create bridge";
-        requiredBy = [ "tor.service" ];
-        before = [ "tor.service" ];
-        script = ''
-          ${ip} link add name br0 type bridge
-          ${ip} link set br0 up
-          ${ip} addr add ${bridgeIp}/24 brd + dev br0
-          ${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
-        '';
-        preStop = ''
-          ${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
-          ${ip} link del br0
-        '';
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = "yes";
-        };
-      };
-
-      bitcoind-import-banlist.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-bitcoind";
-    } //
-    (let
-      makeNetnsServices = n: v: let
-        vethName = "nb-veth-${toString v.id}";
-        netnsName = "nb-${n}";
-        ipNetns = "${ip} -n ${netnsName}";
-        netnsIptables = "${ip} netns exec ${netnsName} ${config.networking.firewall.package}/bin/iptables";
-      in {
-        "${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}";
-
-        "netns-${n}" = rec {
-          requires = [ "netns-bridge.service" ];
-          after = [ "netns-bridge.service" ];
-          bindsTo = [ "${n}.service" ];
-          requiredBy = bindsTo;
-          before = bindsTo;
-          script = ''
-            ${ip} netns add ${netnsName}
-            ${ipNetns} link set lo up
-            ${ip} link add ${vethName} type veth peer name br-${vethName}
-            ${ip} link set ${vethName} netns ${netnsName}
-            ${ipNetns} addr add ${v.address}/24 dev ${vethName}
-            ${ip} link set br-${vethName} up
-            ${ipNetns} link set ${vethName} up
-            ${ip} link set br-${vethName} master br0
-            ${ipNetns} route add default via ${bridgeIp}
-            ${netnsIptables} -w -P INPUT DROP
-            ${netnsIptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
-          '' + (optionalString (config.services.${n}.enforceTor or false)) ''
-            ${netnsIptables} -w -P OUTPUT DROP
-            ${netnsIptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
-          '' + concatMapStrings (otherNetns: let
-            other = netns.${otherNetns};
-          in ''
-            ${netnsIptables} -w -A INPUT -s ${other.address} -j ACCEPT
-            ${netnsIptables} -w -A OUTPUT -d ${other.address} -j ACCEPT
-          '') v.availableNetns;
-          preStop = ''
-            ${ip} netns delete ${netnsName}
-            ${ip} link del br-${vethName}
-          '';
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = "yes";
-            ExecStartPre = "-${ip} netns delete ${netnsName}";
-          };
-        };
-      };
-    in foldl (services: n:
-      services // (makeNetnsServices n netns.${n})
-    ) {} (builtins.attrNames netns));
-
-    # bitcoin: Custom netns configs
     services.bitcoind = {
       bind = netns.bitcoind.address;
       rpcbind = [
@@ -215,22 +240,21 @@ in {
       ];
       rpcallowip = [
         "127.0.0.1"
-      ] ++ lib.lists.concatMap (s: [
-        "${netns.${s}.address}"
-      ]) netns.bitcoind.availableNetns;
-      cli = pkgs.writeScriptBin "bitcoin-cli" ''
-        netns-exec nb-bitcoind ${config.services.bitcoind.package}/bin/bitcoin-cli -datadir='${config.services.bitcoind.dataDir}' "$@"
+      ] ++ map (n: "${netns.${n}.address}") netns.bitcoind.availableNetns;
+      cli = let
+        inherit (config.services.bitcoind) cliBase;
+      in pkgs.writeScriptBin cliBase.name ''
+        exec netns-exec ${netns.bitcoind.netnsName} ${cliBase}/bin/${cliBase.name} "$@"
       '';
     };
+    systemd.services.bitcoind-import-banlist.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-bitcoind";
 
-    # clightning: Custom netns configs
-    services.clightning = mkIf config.services.clightning.enable {
+    services.clightning = {
       bitcoin-rpcconnect = netns.bitcoind.address;
       bind-addr = netns.clightning.address;
     };
 
-    # lnd: Custom netns configs
-    services.lnd = mkIf config.services.lnd.enable {
+    services.lnd = {
       listen = netns.lnd.address;
       rpclisten = [
         "${netns.lnd.address}"
@@ -241,16 +265,10 @@ in {
         "127.0.0.1"
       ];
       bitcoind-host = netns.bitcoind.address;
-      cli = pkgs.writeScriptBin "lncli"
-      # Switch user because lnd makes datadir contents readable by user only
-      ''
-        netns-exec nb-lnd sudo -u lnd ${config.services.lnd.package}/bin/lncli --tlscertpath ${config.nix-bitcoin.secretsDir}/lnd-cert \
-          --macaroonpath '${config.services.lnd.dataDir}/chain/bitcoin/mainnet/admin.macaroon' "$@"
-      '';
+      cliExec = mkCliExec "lnd";
     };
 
-    # liquidd: Custom netns configs
-    services.liquidd = mkIf config.services.liquidd.enable {
+    services.liquidd = {
       bind = netns.liquidd.address;
       rpcbind = [
         "${netns.liquidd.address}"
@@ -258,49 +276,29 @@ in {
       ];
       rpcallowip = [
         "127.0.0.1"
-      ] ++ lib.lists.concatMap (s: [
-        "${netns.${s}.address}"
-      ]) netns.liquidd.availableNetns;
+      ] ++ map (n: "${netns.${n}.address}") netns.liquidd.availableNetns;
       mainchainrpchost = netns.bitcoind.address;
-      cli = pkgs.writeScriptBin "elements-cli" ''
-        netns-exec nb-liquidd ${pkgs.nix-bitcoin.elementsd}/bin/elements-cli -datadir='${config.services.liquidd.dataDir}' "$@"
-      '';
-      swap-cli = pkgs.writeScriptBin "liquidswap-cli" ''
-        netns-exec nb-liquidd ${pkgs.nix-bitcoin.liquid-swap}/bin/liquidswap-cli -c '${config.services.liquidd.dataDir}/elements.conf' "$@"
-      '';
+      cliExec = mkCliExec "liquidd";
     };
 
-    # electrs: Custom netns configs
-    services.electrs = mkIf config.services.electrs.enable {
+    services.electrs = {
       address = netns.electrs.address;
       daemonrpc = "${netns.bitcoind.address}:${toString config.services.bitcoind.rpc.port}";
     };
 
-    # spark-wallet: Custom netns configs
-    services.spark-wallet = mkIf config.services.spark-wallet.enable {
+    services.spark-wallet = {
       host = netns.spark-wallet.address;
       extraArgs = "--no-tls";
     };
 
-    # lightning-charge: Custom netns configs
-    services.lightning-charge.host = mkIf config.services.lightning-charge.enable netns.lightning-charge.address;
+    services.lightning-charge.host = netns.lightning-charge.address;
 
-    # nanopos: Custom netns configs
-    services.nanopos = mkIf config.services.nanopos.enable {
+    services.nanopos = {
       charged-url = "http://${netns.lightning-charge.address}:9112";
       host = netns.nanopos.address;
     };
 
-    # nginx: Custom netns configs
-    services.nix-bitcoin-webindex.host = mkIf config.services.nix-bitcoin-webindex.enable netns.nginx.address;
-
-    # loop: Custom netns configs
-    services.lightning-loop = mkIf config.services.lightning-loop.enable {
-      cli = pkgs.writeScriptBin "loop"
-      # Switch user because lnd makes datadir contents readable by user only
-      ''
-        netns-exec nb-lightning-loop sudo -u lnd ${config.services.lightning-loop.package}/bin/loop "$@"
-      '';
-    };
-  };
+    services.lightning-loop.cliExec = mkCliExec "lightning-loop";
+  }
+  ]);
 }

modules/nix-bitcoin-services.nix

@@ -55,4 +55,11 @@ with lib;
     set -eo pipefail
     ${src}
   '';
+
+  cliExec = mkOption {
+    # Used by netns-isolation to execute the cli in the service's private netns
+    internal = true;
+    type = types.str;
+    default = "exec";
+  };
 }

modules/nix-bitcoin-webindex.nix

@@ -41,7 +41,10 @@ in {
     };
     host = mkOption {
       type = types.str;
-      default = "localhost";
+      default = if config.nix-bitcoin.netns-isolation.enable then
+        config.nix-bitcoin.netns-isolation.netns.nginx.address
+      else
+        "localhost";
       description = "HTTP server listen address.";
     };
     enforceTor =  nix-bitcoin-services.enforceTor;
@@ -79,11 +82,10 @@ in {
       wantedBy = [ "multi-user.target" ];
       path = with pkgs; [
         config.programs.nodeinfo
-        config.services.clightning.cli
-        config.services.lnd.cli
         jq
         sudo
-      ];
+      ] ++ optional config.services.lnd.enable config.services.lnd.cli
+        ++ optional config.services.clightning.enable config.services.clightning.cli;
       serviceConfig = nix-bitcoin-services.defaultHardening // {
         ExecStart="${pkgs.bash}/bin/bash ${createWebIndex}";
         User = "root";

modules/presets/secure-node.nix

@@ -194,7 +194,9 @@ in {
       port = 50001;
       enforceTor = true;
     };
-    services.tor.hiddenServices.electrs = mkHiddenService { port = cfg.electrs.port; toHost = cfg.electrs.address; };
+    services.tor.hiddenServices.electrs = mkIf cfg.electrs.enable (mkHiddenService {
+      port = cfg.electrs.port; toHost = cfg.electrs.address;
+    });
 
     services.spark-wallet = {
       onion-service = true;
@@ -236,6 +238,7 @@ in {
             [ cfg.hardware-wallets.group ]);
       openssh.authorizedKeys.keys = config.users.users.root.openssh.authorizedKeys.keys;
     };
+    nix-bitcoin.netns-isolation.allowedUser = operatorName;
     # Give operator access to onion hostnames
     services.onion-chef.enable = true;
     services.onion-chef.access.${operatorName} = [ "bitcoind" "clightning" "nginx" "liquidd" "spark-wallet" "electrs" "sshd" ];

pkgs/netns-exec/default.nix

@@ -5,7 +5,6 @@ stdenv.mkDerivation {
   buildInputs = [ pkgs.libcap ];
   src = ./src;
   installPhase = ''
-    mkdir -p $out
-    cp main $out/netns-exec
+    cp main $out
   '';
 }

pkgs/netns-exec/src/main.c

@@ -1,7 +1,4 @@
-/* This program must be run with CAP_SYS_ADMIN. This can be achieved for example
- * with
- *         # setcap CAP_SYS_ADMIN+ep ./main
- */
+/* This program requires CAP_SYS_ADMIN */
 
 #define _GNU_SOURCE
 #include <sched.h>
@@ -12,18 +9,17 @@
 #include <fcntl.h>
 #include <sys/capability.h>
 
-static char *available_netns[] = {
+static char *allowed_netns[] = {
     "nb-lnd",
     "nb-lightning-loop",
     "nb-bitcoind",
     "nb-liquidd"
 };
 
-int check_netns(char *netns) {
-    int i;
-    int n_available_netns = sizeof(available_netns) / sizeof(available_netns[0]);
-    for (i = 0; i < n_available_netns; i++) {
-        if (strcmp(available_netns[i], netns) == 0) {
+int is_netns_allowed(char *netns) {
+    int n_allowed_netns = sizeof(allowed_netns) / sizeof(allowed_netns[0]);
+    for (int i = 0; i < n_allowed_netns; i++) {
+        if (strcmp(allowed_netns[i], netns) == 0) {
             return 1;
         }
     }
@@ -35,6 +31,7 @@ void print_capabilities() {
     printf("Capabilities: %s\n", cap_to_text(caps, NULL));
     cap_free(caps);
 }
+
 void drop_capabilities() {
     cap_t caps = cap_get_proc();
     cap_clear(caps);
@@ -43,25 +40,24 @@ void drop_capabilities() {
 }
 
 int main(int argc, char **argv) {
-    int fd;
     char netns_path[256];
 
     if (argc < 3) {
-        printf("usage: %s <netns> <command to execute>\n", argv[0]);
+        printf("usage: %s <netns> <command>\n", argv[0]);
         return 1;
     }
 
-    if (!check_netns(argv[1])) {
-        printf("Failed checking %s against available netns.\n", argv[1]);
+    if (!is_netns_allowed(argv[1])) {
+        printf("%s is not an allowed netns.\n", argv[1]);
         return 1;
     }
 
     if(snprintf(netns_path, sizeof(netns_path), "/var/run/netns/%s", argv[1]) < 0) {
-        printf("Failed concatenating %s to the netns path.\n", argv[1]);
+        printf("Path length exceeded for netns %s.\n", argv[1]);
         return 1;
     }
 
-    fd = open(netns_path, O_RDONLY);
+    int fd = open(netns_path, O_RDONLY);
     if (fd < 0) {
         printf("Failed opening netns %s: %d, %s \n", netns_path, errno, strerror(errno));
         return 1;
@@ -84,4 +80,3 @@ int main(int argc, char **argv) {
     execvp(argv[2], &argv[2]);
     return 0;
 }
-

test/base.py

@@ -1,3 +1,6 @@
+is_interactive = "is_interactive" in vars()
+
+
 def succeed(*cmds):
     """Returns the concatenated output of all cmds"""
     return machine.succeed(*cmds)
@@ -29,15 +32,15 @@ def assert_running(unit):
     assert_no_failure(unit)
 
 
+def run_tests(extra_tests):
+    """
+    :param extra_tests: Test functions that hook into the testing code below
+    :type extra_tests: Dict[str, Callable[]]
+    """
     # Don't execute the following test suite when this script is running in interactive mode
-if "is_interactive" in vars():
+    if is_interactive:
         raise Exception()
 
-### Tests
-
-# The argument extra_tests is a dictionary from strings to functions. The string
-# determines at which point of run_tests the corresponding function is executed.
-def run_tests(extra_tests):
     test_security()
 
     assert_running("bitcoind")

test/run-tests.sh

@@ -4,53 +4,55 @@
 # The test (./test.nix) uses the NixOS testing framework and is executed in a VM.
 #
 # Usage:
-#   Run test
+#   Run all tests
+#   ./run-tests.sh
+#
+#   Test specific scenario
 #   ./run-tests.sh --scenario <scenario>
 #
-#   Run test and save result to avoid garbage collection
-#   ./run-tests.sh --scenario <scenario> build --out-link /tmp/nix-bitcoin-test
+#   Run test and link results to avoid garbage collection
+#   ./run-tests.sh [--scenario <scenario>] --out-link-prefix /tmp/nix-bitcoin-test build
+#
+#   Pass extra args to nix-build
+#   ./run-tests.sh build --builders 'ssh://mybuildhost - - 15'
 #
 #   Run interactive test debugging
-#   ./run-tests.sh --scenario <scenario> debug
+#   ./run-tests.sh [--scenario <scenario>] debug
 #
 #   This starts the testing VM and drops you into a Python REPL where you can
 #   manually execute the tests from ./test-script.py
 
 set -eo pipefail
 
-die() {
-    printf '%s\n' "$1" >&2
-    exit 1
-}
-
-# Initialize all the option variables.
-# This ensures we are not contaminated by variables from the environment.
 scenario=
-
+outLinkPrefix=
 while :; do
     case $1 in
-	--scenario)
-	    if [ "$2" ]; then
+        --scenario|-s)
+            if [[ $2 ]]; then
                 scenario=$2
                 shift
+                shift
             else
-		die 'ERROR: "--scenario" requires a non-empty option argument.'
+                >&2 echo 'Error: "$1" requires an argument.'
+                exit 1
             fi
             ;;
-	-?*)
-	    printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2
+        --out-link-prefix|-o)
+            if [[ $2 ]]; then
+                outLinkPrefix=$2
+                shift
+                shift
+            else
+                >&2 echo 'Error: "$1" requires an argument.'
+                exit 1
+            fi
             ;;
         *)
             break
     esac
-
-    shift
 done
 
-if [[ -z $scenario ]]; then
-    die 'ERROR: "--scenario" is required'
-fi
-
 numCPUs=${numCPUs:-$(nproc)}
 # Min. 800 MiB needed to avoid 'out of memory' errors
 memoryMiB=${memoryMiB:-2048}
@@ -108,8 +110,13 @@ debug() {
 }
 
 # Run the test by building the test derivation
-build() {
-    vmTestNixExpr | nix-build --no-out-link "$@" -
+buildTest() {
+    if [[ $outLinkPrefix ]]; then
+        buildArgs="--out-link $outLinkPrefix-$scenario"
+    else
+        buildArgs=--no-out-link
+    fi
+    vmTestNixExpr | nix-build $buildArgs "$@" -
 }
 
 # On continuous integration nodes there are few other processes running alongside the
@@ -137,4 +144,18 @@ vmTestNixExpr() {
 EOF
 }
 
+build() {
+    if [[ $scenario ]]; then
+        buildTest "$@"
+    else
+        scenario=default buildTest "$@"
+        scenario=withnetns buildTest "$@"
+    fi
+}
+
+# Set default scenario for all actions other than 'build'
+if [[ $1 && $1 != build ]]; then
+    : ${scenario:=default}
+fi
+
 eval "${@:-build}"

test/test.nix

@@ -1,14 +1,10 @@
 # Integration test, can be run without internet access.
 
+# Make sure to update build() in ./run-tests.sh when adding new scenarios
 { scenario ? "default" }:
 
-let
-  netns-isolation = builtins.getAttr scenario { default = false; withnetns = true; };
-  testScriptFilename = builtins.getAttr scenario { default = ./scenarios/default.py; withnetns = ./scenarios/withnetns.py; };
-in
-
 import ./make-test.nix rec {
-  name = "nix-bitcoin";
+  name = "nix-bitcoin-${scenario}";
 
   hardened = {
     imports = [ <nixpkgs/nixos/modules/profiles/hardened.nix> ];
@@ -23,7 +19,7 @@ import ./make-test.nix rec {
       # hardened
     ];
 
-    nix-bitcoin.netns-isolation.enable = mkForce netns-isolation;
+    nix-bitcoin.netns-isolation.enable = (scenario == "withnetns");
 
     services.bitcoind.extraConfig = mkForce "connect=0";
 
@@ -61,5 +57,6 @@ import ./make-test.nix rec {
       install -o nobody -g nogroup -m777 <(:) /secrets/dummy
     '';
   };
-  testScript = builtins.readFile ./scenarios/lib.py + "\n\n" + builtins.readFile testScriptFilename;
+  testScript =
+    builtins.readFile ./base.py + "\n\n" + builtins.readFile "${./.}/scenarios/${scenario}.py";
 }