Merge #187: Network Namespaces

681728294b tests: run scenarios as multiple Travis jobs (nixbitcoin)
43ce847e2b tests: allow running integration tests with different configurations (nixbitcoin)
8783f38fba tests: add netns to testing framework (nixbitcoin)
25adce29e5 secure-node: only mkHiddenServices if services are enabled (nixbitcoin)
c542b92e55 nginx: add netns (nixbitcoin)
ef89607704 recurring-donations: add netns (nixbitcoin)
582cb86d74 nanopos: add netns (nixbitcoin)
7369f0a7ec lightning-charge: add netns (nixbitcoin)
c4ab73d51f spark-wallet: add netns (nixbitcoin)
d6296acaba electrs: add netns (nixbitcoin)
c0b02ac93a liquid: add netns cli script (nixbitcoin)
672a416ede liquidd: add netns (nixbitcoin)
4b8ca52647 lnd: add netns cli script (nixbitcoin)
c55296433d lnd: add netns (nixbitcoin)
f3d2aaa5d4 lnd: prepare for netns and bring in line with clightning (nixbitcoin)
3c0c446547 clightning: add netns (nixbitcoin)
ae1230e13b clightning: remove bitcoin-rpcuser option (nixbitcoin)
65b5dab3d4 clightning: add announce-tor (nixbitcoin)
515aae2825 bitcoind: add netns and nonetns cli scripts (nixbitcoin)
75ca6f186c bitcoind: add netns (nixbitcoin)
e5e07b91f7 netns-isolation: netns architecture (nixbitcoin)
4a7199a3da netns-exec: add c program to execute commands in netns (Jonas Nick)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 681728294b

Tree-SHA512: 53083067dd94bfb38d06e9b7f0d2deaeb1b95067dc0c6e50d07073405ff5270de54fa56761e861d23eb0f0a9457f0e1e2c46ed71d732c0cfbaa09b6d425e0691

.travis.yml

@@ -14,7 +14,8 @@ env:
     # CACHIX_SIGNING_KEY
   - secure: "xXCFZ7g+k5YmCGm8R8l3bZElVmt+RD1KscG3kGr5w4HyyDPTzFetPo+sT8bUpysDU0u3HWhfVhHtpog2mhNhwVl3tQwKXea3dHKC1i6ypBg3gjDngmJRR5wo++ocYDpK8qPaU7m/jHQTNFnTA4CbmMcc05GcYx/1Ai/ZGkNwWFjdIcVeOUoiol33gykMOXIGDg2qlXudt33wP53FHbX8L4fxzodWfAuxKK4AoGprxy5eSnU7LCaXxxJmu4HwuV+Ux2U1NfE/E33cvhlUvTQCswVSZFG06mg8rwhMG1ozsDvlL2itZlu/BeUQH5y3XMMlnJIUXUazkRBibf1w/ebVjpOF+anqkqmq8tcbFEa7T+RJeVTIsvP+L8rE8fcmuZtdg9hNmgRnLmaeT0vVwD1L2UqW9HdRyujdoS0jPYuoc1W7f1JQWfAPhBPQ1SrtKyNNqcbVJ34aN7b+4vCzRpQL1JTbmjzQIWhkiKN1qMo1v/wbIydW8yka4hc4JOfdQLaAJEPI1eAC1MLotSAegMnwKWE1dzm66MuPSipksYjZrvsB28cV4aCVUffIuRhrSr1i2afRHwTpNbK9U4/576hah15ftUdR79Sfkcoi1ekSQTFGRvkRIPYtkKLYwFa3jVA41qz7+IIZCf4TsApy3XDdFx91cRub7yPq9BeZ83A+qYQ="
   jobs:
-  - TestModules=1 STABLE=1
+  - TestModules=1 STABLE=1 SCENARIO=default
+  - TestModules=1 STABLE=1 SCENARIO=withnetns
   - PKG=hwi STABLE=1
   - PKG=hwi STABLE=0
   - PKG=lightning-charge STABLE=1
@@ -38,7 +39,7 @@ script:
                 exit 1
             fi
             sudo chmod go+rw /dev/kvm
-            test/run-tests.sh exprForCI
+            test/run-tests.sh --scenario $SCENARIO exprForCI
         else
             echo "(import ./. {}).$PKG"
         fi

examples/configuration.nix

@@ -38,14 +38,19 @@
   # Enable this module to use clightning, a Lightning Network implementation
   # in C.
   services.clightning.enable = true;
-  # Enable this option to listen for incoming lightning connections. By
+  # Enable this option to announce our Tor Hidden Service. By default clightning
-  # default nix-bitcoin nodes offer outgoing connectivity.
+  # offers outgoing functionality, but doesn't announce the Tor Hidden Service
-  # services.clightning.autolisten = true;
+  # under which peers can reach us.
+  # services.clightning.announce-tor = true;
 
   ### LND
   # Disable clightning and uncomment the following line in order to enable lnd,
   # a lightning implementation written in Go.
   # services.lnd.enable = true;
+  # Enable this option to announce our Tor Hidden Service. By default lnd
+  # offers outgoing functionality, but doesn't announce the Tor Hidden Service
+  # under which peers can reach us.
+  # services.lnd.announce-tor = true;
   ## WARNING
   # If you use lnd, you should manually backup your wallet mnemonic
   # seed. This will allow you to recover on-chain funds. You can run the
@@ -117,6 +122,13 @@
   # `docs/usage.md`.
   # services.hardware-wallets.trezor = true;
 
+  ### netns-isolation (EXPERIMENTAL)
+  # Enable this module to use Network Namespace Isolation. This feature places
+  # every service in its own network namespace and only allows truly necessary
+  # connections between network namespaces, making sure services are isolated on
+  # a network-level as much as possible.
+  # nix-bitcoin.netns-isolation.enable = true;
+
   # FIXME: Define your hostname.
   networking.hostName = "nix-bitcoin";
   time.timeZone = "UTC";

modules/bitcoind.nix

@@ -18,6 +18,7 @@ let
     ${optionalString (cfg.assumevalid != null) "assumevalid=${cfg.assumevalid}"}
 
     # Connection options
+    ${optionalString cfg.listen "bind=${cfg.bind}"}
     ${optionalString (cfg.port != null) "port=${toString cfg.port}"}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     listen=${if cfg.listen then "1" else "0"}
@@ -30,6 +31,8 @@ let
       (rpcUser: "rpcauth=${rpcUser.name}:${rpcUser.passwordHMAC}")
       (attrValues cfg.rpc.users)
     }
+    ${lib.concatMapStrings (rpcbind: "rpcbind=${rpcbind}\n") cfg.rpcbind}
+    ${lib.concatMapStrings (rpcallowip: "rpcallowip=${rpcallowip}\n") cfg.rpcallowip}
     ${optionalString (cfg.rpcuser != null) "rpcuser=${cfg.rpcuser}"}
     ${optionalString (cfg.rpcpassword != null) "rpcpassword=${cfg.rpcpassword}"}
 
@@ -68,6 +71,13 @@ in {
         default = "/var/lib/bitcoind";
         description = "The data directory for bitcoind.";
       };
+      bind = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = ''
+          Bind to given address and always listen on it.
+        '';
+      };
       user = mkOption {
         type = types.str;
         default = "bitcoin";
@@ -117,6 +127,20 @@ in {
           '';
         };
       };
+      rpcbind = mkOption {
+        type = types.listOf types.str;
+        default = [ "127.0.0.1" ];
+        description = ''
+          Bind to given address to listen for JSON-RPC connections.
+        '';
+      };
+      rpcallowip = mkOption {
+        type = types.listOf types.str;
+        default = [ "127.0.0.1" ];
+        description = ''
+          Allow JSON-RPC connections from specified source.
+        '';
+      };
       rpcuser = mkOption {
           type = types.nullOr types.str;
           default = "bitcoinrpc";
@@ -233,11 +257,20 @@ in {
       };
       cli = mkOption {
         type = types.package;
+        default = cfg.cli-nonetns-exec;
+        description = "Binary to connect with the bitcoind instance.";
+      };
+      # Needed because bitcoind-import-banlist already executes inside
+      # nb-bitcoind, hence it doesn't need netns-exec prefixed.
+      cli-nonetns-exec = mkOption {
         readOnly = true;
+        type = types.package;
         default = pkgs.writeScriptBin "bitcoin-cli" ''
           exec ${cfg.package}/bin/bitcoin-cli -datadir='${cfg.dataDir}' "$@"
         '';
-        description = "Binary to connect with the bitcoind instance.";
+        description = ''
+          Binary to connect with the bitcoind instance without netns-exec.
+        '';
       };
       enforceTor =  nix-bitcoin-services.enforceTor;
     };
@@ -297,7 +330,7 @@ in {
       bindsTo = [ "bitcoind.service" ];
       after = [ "bitcoind.service" ];
       script = ''
-        cd ${cfg.cli}/bin
+        cd ${cfg.cli-nonetns-exec}/bin
         # Poll until bitcoind accepts commands. This can take a long time.
         while ! ./bitcoin-cli getnetworkinfo &> /dev/null; do
           sleep 1

modules/clightning.nix

@@ -5,14 +5,15 @@ with lib;
 let
   cfg = config.services.clightning;
   inherit (config) nix-bitcoin-services;
+  onion-chef-service = (if cfg.announce-tor then [ "onion-chef.service" ] else []);
   configFile = pkgs.writeText "config" ''
-    autolisten=${if cfg.autolisten then "true" else "false"}
     network=bitcoin
     bitcoin-datadir=${config.services.bitcoind.dataDir}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     always-use-proxy=${if cfg.always-use-proxy then "true" else "false"}
     ${optionalString (cfg.bind-addr != null) "bind-addr=${cfg.bind-addr}"}
-    bitcoin-rpcuser=${cfg.bitcoin-rpcuser}
+    ${optionalString (cfg.bitcoin-rpcconnect != null) "bitcoin-rpcconnect=${cfg.bitcoin-rpcconnect}"}
+    bitcoin-rpcuser=${config.services.bitcoind.rpcuser}
     rpc-file-mode=0660
   '';
 in {
@@ -28,7 +29,8 @@ in {
       type = types.bool;
       default = false;
       description = ''
-        If enabled, the clightning service will listen.
+        Bind (and maybe announce) on IPv4 and IPv6 interfaces if no addr,
+        bind-addr or  announce-addr  options  are specified.
       '';
     };
     proxy = mkOption {
@@ -48,11 +50,15 @@ in {
       default = null;
       description = "Set an IP address or UNIX domain socket to listen to";
     };
-    bitcoin-rpcuser = mkOption {
+    announce-tor = mkOption {
-      type = types.str;
+      type = types.bool;
-      description = ''
+      default = false;
-        Bitcoin RPC user
+      description = "Announce clightning Tor Hidden Service";
-      '';
+    };
+    bitcoin-rpcconnect = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "The bitcoind RPC host to connect to.";
     };
     dataDir = mkOption {
       type = types.path;
@@ -93,12 +99,13 @@ in {
       "d '${cfg.dataDir}' 0770 ${cfg.user} ${cfg.group} - -"
     ];
 
+    services.onion-chef.access.clightning = if cfg.announce-tor then [ "clightning" ] else [];
     systemd.services.clightning = {
       description = "Run clightningd";
       path  = [ pkgs.nix-bitcoin.bitcoind ];
       wantedBy = [ "multi-user.target" ];
-      requires = [ "bitcoind.service" ];
+      requires = [ "bitcoind.service" ] ++ onion-chef-service;
-      after = [ "bitcoind.service" ];
+      after = [ "bitcoind.service" ] ++ onion-chef-service;
       preStart = ''
         cp ${configFile} ${cfg.dataDir}/config
         chown -R '${cfg.user}:${cfg.group}' '${cfg.dataDir}'
@@ -106,6 +113,7 @@ in {
         rm -f ${cfg.dataDir}/bitcoin/lightning-rpc
         chmod 600 ${cfg.dataDir}/config
         echo "bitcoin-rpcpassword=$(cat ${config.nix-bitcoin.secretsDir}/bitcoin-rpcpassword)" >> '${cfg.dataDir}/config'
+        ${optionalString cfg.announce-tor "echo announce-addr=$(cat /var/lib/onion-chef/clightning/clightning) >> '${cfg.dataDir}/config'"}
         '';
       serviceConfig = nix-bitcoin-services.defaultHardening // {
         ExecStart = "${pkgs.nix-bitcoin.clightning}/bin/lightningd --lightning-dir=${cfg.dataDir}";

modules/electrs.nix

@@ -17,6 +17,14 @@ in {
       default = "/var/lib/electrs";
       description = "The data directory for electrs.";
     };
+    # Needed until electrs tls proxy is removed
+    host = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = ''
+        The host on which incoming connections arrive.
+      '';
+    };
     user = mkOption {
       type = types.str;
       default = "electrs";
@@ -44,6 +52,13 @@ in {
       default = 50001;
       description = "RPC port.";
     };
+    daemonrpc = mkOption {
+      type = types.str;
+      default = "127.0.0.1:8332";
+      description = ''
+        Bitcoin daemon JSONRPC 'addr:port' to connect
+      '';
+    };
     extraArgs = mkOption {
       type = types.separatedString " ";
       default = "";
@@ -97,7 +112,8 @@ in {
               "--jsonrpc-import --index-batch-size=10"
           } \
           --db-dir '${cfg.dataDir}' --daemon-dir '${config.services.bitcoind.dataDir}' \
-          --electrum-rpc-addr=${toString cfg.address}:${toString cfg.port} ${cfg.extraArgs}
+          --electrum-rpc-addr=${toString cfg.address}:${toString cfg.port} \
+          --daemon-rpc-addr=${toString cfg.daemonrpc} ${cfg.extraArgs}
         '';
         User = cfg.user;
         Group = cfg.group;

modules/lightning-charge.nix

@@ -21,6 +21,17 @@ in {
       default = "/var/lib/lightning-charge";
       description = "The data directory for lightning-charge.";
     };
+    host = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = "http server listen address";
+    };
+    extraArgs = mkOption {
+      type = types.separatedString " ";
+      default = "";
+      description = "Extra command line arguments passed to lightning-charge.";
+    };
+    enforceTor =  nix-bitcoin-services.enforceTor;
   };
 
   config = mkIf cfg.enable {
@@ -60,13 +71,15 @@ in {
           # Needed to access clightning.dataDir in preStart
           PermissionsStartOnly = "true";
           EnvironmentFile = "${config.nix-bitcoin.secretsDir}/lightning-charge-env";
-          ExecStart = "${pkgs.nix-bitcoin.lightning-charge}/bin/charged -l ${config.services.clightning.dataDir}/bitcoin -d ${cfg.dataDir}/lightning-charge.db";
+          ExecStart = "${pkgs.nix-bitcoin.lightning-charge}/bin/charged -l ${config.services.clightning.dataDir}/bitcoin -d ${cfg.dataDir}/lightning-charge.db -i ${cfg.host} ${cfg.extraArgs}";
           User = user;
           Restart = "on-failure";
           RestartSec = "10s";
           ReadWritePaths = "${cfg.dataDir}";
-      } // nix-bitcoin-services.nodejs
+      } // (if cfg.enforceTor
-        // nix-bitcoin-services.allowTor;
+            then nix-bitcoin-services.allowTor
+            else nix-bitcoin-services.allowAnyIP)
+        // nix-bitcoin-services.nodejs;
     };
     nix-bitcoin.secrets.lightning-charge-env.user = user;
   };

modules/liquid.nix

@@ -15,6 +15,7 @@ let
     ${optionalString (cfg.validatepegin != null) "validatepegin=${if cfg.validatepegin then "1" else "0"}"}
 
     # Connection options
+    ${optionalString cfg.listen "bind=${cfg.bind}"}
     ${optionalString (cfg.port != null) "port=${toString cfg.port}"}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     listen=${if cfg.listen then "1" else "0"}
@@ -25,8 +26,11 @@ let
       (rpcUser: "rpcauth=${rpcUser.name}:${rpcUser.passwordHMAC}")
       (attrValues cfg.rpc.users)
     }
+    ${lib.concatMapStrings (rpcbind: "rpcbind=${rpcbind}\n") cfg.rpcbind}
+    ${lib.concatMapStrings (rpcallowip: "rpcallowip=${rpcallowip}\n") cfg.rpcallowip}
     ${optionalString (cfg.rpcuser != null) "rpcuser=${cfg.rpcuser}"}
     ${optionalString (cfg.rpcpassword != null) "rpcpassword=${cfg.rpcpassword}"}
+    ${optionalString (cfg.mainchainrpchost != null) "mainchainrpchost=${cfg.mainchainrpchost}"}
 
     # Extra config options (from liquidd nixos service)
     ${cfg.extraConfig}
@@ -80,6 +84,13 @@ in {
         default = "/var/lib/liquidd";
         description = "The data directory for liquidd.";
       };
+      bind = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = ''
+          Bind to given address and always listen on it.
+        '';
+      };
 
       user = mkOption {
         type = types.str;
@@ -111,6 +122,20 @@ in {
         };
       };
 
+      rpcbind = mkOption {
+        type = types.listOf types.str;
+        default = [ "127.0.0.1" ];
+        description = ''
+          Bind to given address to listen for JSON-RPC connections.
+        '';
+      };
+      rpcallowip = mkOption {
+        type = types.listOf types.str;
+        default = [ "127.0.0.1" ];
+        description = ''
+          Allow JSON-RPC connections from specified source.
+        '';
+      };
       rpcuser = mkOption {
           type = types.nullOr types.str;
           default = null;
@@ -121,6 +146,14 @@ in {
           default = null;
           description = "Password for JSON-RPC connections";
       };
+      mainchainrpchost = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          The address which the daemon will try to connect to the trusted
+          mainchain daemon to validate peg-ins.
+        '';
+      };
 
       testnet = mkOption {
         type = types.bool;
@@ -177,14 +210,12 @@ in {
         '';
       };
       cli = mkOption {
-        readOnly = true;
         default = pkgs.writeScriptBin "elements-cli" ''
           exec ${pkgs.nix-bitcoin.elementsd}/bin/elements-cli -datadir='${cfg.dataDir}' "$@"
         '';
         description = "Binary to connect with the liquidd instance.";
       };
       swap-cli = mkOption {
-        readOnly = true;
         default = pkgs.writeScriptBin "liquidswap-cli" ''
           exec ${pkgs.nix-bitcoin.liquid-swap}/bin/liquidswap-cli -c '${cfg.dataDir}/elements.conf' "$@"
         '';

modules/lnd.nix

@@ -5,6 +5,7 @@ with lib;
 let
   cfg = config.services.lnd;
   inherit (config) nix-bitcoin-services;
+  onion-chef-service = (if cfg.announce-tor then [ "onion-chef.service" ] else []);
   secretsDir = config.nix-bitcoin.secretsDir;
   configFile = pkgs.writeText "lnd.conf" ''
     datadir=${cfg.dataDir}
@@ -13,17 +14,17 @@ let
     tlscertpath=${secretsDir}/lnd-cert
     tlskeypath=${secretsDir}/lnd-key
 
-    rpclisten=localhost:${toString cfg.rpcPort}
+    listen=${toString cfg.listen}
-    restlisten=localhost:${toString cfg.restPort}
+    ${lib.concatMapStrings (rpclisten: "rpclisten=${rpclisten}:${toString cfg.rpcPort}\n") cfg.rpclisten}
+    ${lib.concatMapStrings (restlisten: "restlisten=${restlisten}:${toString cfg.restPort}\n") cfg.restlisten}
 
     bitcoin.active=1
     bitcoin.node=bitcoind
 
     tor.active=true
-    tor.v3=true
+    ${optionalString (cfg.tor-socks != null) "tor.socks=${cfg.tor-socks}"}
-    tor.streamisolation=true
-    tor.privatekeypath=${cfg.dataDir}/v3_onion_private_key
 
+    bitcoind.rpchost=${cfg.bitcoind-host}
     bitcoind.rpcuser=${config.services.bitcoind.rpcuser}
     bitcoind.zmqpubrawblock=${config.services.bitcoind.zmqpubrawblock}
     bitcoind.zmqpubrawtx=${config.services.bitcoind.zmqpubrawtx}
@@ -45,6 +46,25 @@ in {
       default = "/var/lib/lnd";
       description = "The data directory for LND.";
     };
+    listen = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = "Bind to given address to listen to peer connections";
+    };
+    rpclisten = mkOption {
+      type = types.listOf types.str;
+      default = [ "localhost" ];
+      description = ''
+        Bind to given address to listen to RPC connections.
+      '';
+    };
+    restlisten = mkOption {
+      type = types.listOf types.str;
+      default = [ "localhost" ];
+      description = ''
+        Bind to given address to listen to REST connections.
+      '';
+    };
     rpcPort = mkOption {
       type = types.port;
       default = 10009;
@@ -55,6 +75,23 @@ in {
       default = 8080;
       description = "Port on which to listen for REST connections.";
     };
+    bitcoind-host = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = ''
+        The host that your local bitcoind daemon is listening on.
+      '';
+    };
+    tor-socks = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "Set a socks proxy to use to connect to Tor nodes";
+    };
+    announce-tor = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Announce LND Tor Hidden Service";
+    };
     extraConfig = mkOption {
       type = types.lines;
       default = "";
@@ -70,7 +107,6 @@ in {
       description = "The package providing lnd binaries.";
     };
     cli = mkOption {
-      readOnly = true;
       default = pkgs.writeScriptBin "lncli"
       # Switch user because lnd makes datadir contents readable by user only
       ''
@@ -96,19 +132,21 @@ in {
     ];
 
     services.bitcoind = {
-      zmqpubrawblock = "tcp://127.0.0.1:28332";
+      zmqpubrawblock = "tcp://${cfg.bitcoind-host}:28332";
-      zmqpubrawtx = "tcp://127.0.0.1:28333";
+      zmqpubrawtx = "tcp://${cfg.bitcoind-host}:28333";
     };
 
+    services.onion-chef.access.lnd = if cfg.announce-tor then [ "lnd" ] else [];
     systemd.services.lnd = {
       description = "Run LND";
       path  = [ pkgs.nix-bitcoin.bitcoind ];
       wantedBy = [ "multi-user.target" ];
-      requires = [ "bitcoind.service" ];
+      requires = [ "bitcoind.service" ] ++ onion-chef-service;
-      after = [ "bitcoind.service" ];
+      after = [ "bitcoind.service" ] ++ onion-chef-service;
       preStart = ''
         install -m600 ${configFile} '${cfg.dataDir}/lnd.conf'
         echo "bitcoind.rpcpass=$(cat ${secretsDir}/bitcoin-rpcpassword)" >> '${cfg.dataDir}/lnd.conf'
+        ${optionalString cfg.announce-tor "echo externalip=$(cat /var/lib/onion-chef/lnd/lnd) >> '${cfg.dataDir}/lnd.conf'"}
       '';
       serviceConfig = nix-bitcoin-services.defaultHardening // {
         ExecStart = "${cfg.package}/bin/lnd --configfile=${cfg.dataDir}/lnd.conf";

modules/modules.nix

@@ -14,6 +14,7 @@
     ./hardware-wallets.nix
     ./lnd.nix
     ./secrets/secrets.nix
+    ./netns-isolation.nix
   ];
 
   disabledModules = [ "services/networking/bitcoind.nix" ];

modules/nanopos.nix

@@ -49,6 +49,26 @@ in {
         "The items file (see nanopos README).";
       '';
     };
+    charged-url = mkOption {
+      type = types.str;
+      default = "http://localhost:9112";
+      description = ''
+        "The lightning charge server url.";
+      '';
+    };
+    host = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = ''
+        "http server listen address.";
+      '';
+    };
+    extraArgs = mkOption {
+      type = types.separatedString " ";
+      default = "";
+      description = "Extra command line arguments passed to nanopos.";
+    };
+    enforceTor =  nix-bitcoin-services.enforceTor;
   };
 
   config = mkIf cfg.enable {
@@ -59,6 +79,20 @@ in {
     ];
 
     environment.systemPackages = [ pkgs.nix-bitcoin.nanopos ];
+
+    services.nginx = {
+      enable = true;
+      virtualHosts."_" = {
+        root = "/var/www";
+        extraConfig = ''
+          location /store/ {
+            proxy_pass http://${toString cfg.host}:${toString cfg.port};
+            rewrite /store/(.*) /$1 break;
+          }
+        '';
+      };
+    };
+
     systemd.services.nanopos = {
       description = "Run nanopos";
       wantedBy = [ "multi-user.target" ];
@@ -66,12 +100,14 @@ in {
       after = [ "lightning-charge.service" ];
       serviceConfig = nix-bitcoin-services.defaultHardening // {
         EnvironmentFile = "${config.nix-bitcoin.secretsDir}/nanopos-env";
-        ExecStart = "${pkgs.nix-bitcoin.nanopos}/bin/nanopos -y ${cfg.itemsFile} -p ${toString cfg.port} --show-bolt11";
+        ExecStart = "${pkgs.nix-bitcoin.nanopos}/bin/nanopos -y ${cfg.itemsFile} -i ${toString cfg.host} -p ${toString cfg.port} -c ${toString cfg.charged-url} --show-bolt11 ${cfg.extraArgs}";
         User = "nanopos";
         Restart = "on-failure";
         RestartSec = "10s";
-      } // nix-bitcoin-services.nodejs
+      } // (if cfg.enforceTor
-        // nix-bitcoin-services.allowTor;
+            then nix-bitcoin-services.allowTor
+            else nix-bitcoin-services.allowAnyIP)
+        // nix-bitcoin-services.nodejs;
     };
     users.users.nanopos = {
       description = "nanopos User";

modules/netns-isolation.nix

@@ -0,0 +1,303 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.nix-bitcoin.netns-isolation;
+
+  netns = builtins.mapAttrs (n: v: {
+    inherit (v) id;
+    address = "169.254.${toString cfg.addressblock}.${toString v.id}";
+    availableNetns = builtins.filter isEnabled availableNetns.${n};
+  }) enabledServices;
+
+  # Symmetric netns connection matrix
+  # if clightning.connections = [ "bitcoind" ]; then
+  #   availableNetns.bitcoind = [ "clighting" ];
+  #   and
+  #   availableNetns.clighting = [ "bitcoind" ];
+  availableNetns = let
+    # base = { clightning = [ "bitcoind" ]; ... }
+    base = builtins.mapAttrs (n: v:
+      builtins.filter isEnabled v.connections
+    ) enabledServices;
+  in
+    foldl (xs: s1:
+      foldl (xs: s2:
+        xs // { "${s2}" = xs.${s2} ++ [ s1 ]; }
+      ) xs cfg.services.${s1}.connections
+    ) base (builtins.attrNames base);
+
+  enabledServices = filterAttrs (n: v: isEnabled n) cfg.services;
+  isEnabled = x: config.services.${x}.enable;
+
+  ip = "${pkgs.iproute}/bin/ip";
+  iptables = "${config.networking.firewall.package}/bin/iptables";
+
+  bridgeIp = "169.254.${toString cfg.addressblock}.10";
+
+in {
+  options.nix-bitcoin.netns-isolation = {
+    enable = mkEnableOption "netns isolation";
+
+    addressblock = mkOption {
+      type = types.ints.u8;
+      default = "1";
+      description = ''
+        Specify the N address block in 169.254.N.0/24.
+      '';
+    };
+
+    services = mkOption {
+      default = {};
+      type = types.attrsOf (types.submodule {
+        options = {
+          id = mkOption {
+            # TODO: Exclude 10
+            # TODO: Assert uniqueness
+            type = types.int;
+            description = ''
+              id for the netns, that is used for the IP address host part and
+              naming the interfaces. Must be unique. Must not be 10.
+            '';
+          };
+          connections = mkOption {
+            type = with types; listOf str;
+            default = [];
+          };
+        };
+      });
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.enable {
+      # Prerequisites
+      networking.dhcpcd.denyInterfaces = [ "br0" "br-nb*" "nb-veth*" ];
+      services.tor.client.socksListenAddress = "${bridgeIp}:9050";
+      networking.firewall.interfaces.br0.allowedTCPPorts = [ 9050 ];
+      boot.kernel.sysctl."net.ipv4.ip_forward" = true;
+      security.wrappers.netns-exec = {
+        source  = "${pkgs.nix-bitcoin.netns-exec}/netns-exec";
+        capabilities = "cap_sys_admin=ep";
+        owner = "${config.nix-bitcoin.operatorName}";
+        permissions = "u+rx,g+rx,o-rwx";
+      };
+
+      nix-bitcoin.netns-isolation.services = {
+        bitcoind = {
+          id = 12;
+        };
+        clightning = {
+          id = 13;
+          connections = [ "bitcoind" ];
+        };
+        lnd = {
+          id = 14;
+          connections = [ "bitcoind" ];
+        };
+        liquidd = {
+          id = 15;
+          connections = [ "bitcoind" ];
+        };
+        electrs = {
+          id = 16;
+          connections = [ "bitcoind" ]
+          ++ ( optionals config.services.electrs.TLSProxy.enable [ "nginx" ]);
+        };
+        spark-wallet = {
+          id = 17;
+          # communicates with clightning over lightning-rpc socket
+          connections = [];
+        };
+        lightning-charge = {
+          id = 18;
+          # communicates with clightning over lightning-rpc socket
+          connections = [];
+        };
+        nanopos = {
+          id = 19;
+          connections = [ "nginx" "lightning-charge" ];
+        };
+        recurring-donations = {
+          id = 20;
+          # communicates with clightning over lightning-rpc socket
+          connections = [];
+        };
+        nginx = {
+          id = 21;
+          connections = [];
+        };
+      };
+
+      systemd.services = {
+        netns-bridge = {
+          description = "Create bridge";
+          requiredBy = [ "tor.service" ];
+          before = [ "tor.service" ];
+          script = ''
+            ${ip} link add name br0 type bridge
+            ${ip} link set br0 up
+            ${ip} addr add ${bridgeIp}/24 brd + dev br0
+            ${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
+          '';
+          preStop = ''
+            ${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
+            ${ip} link del br0
+          '';
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = "yes";
+          };
+        };
+
+        bitcoind-import-banlist.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-bitcoind";
+      } //
+      (let
+        makeNetnsServices = n: v: let
+          vethName = "nb-veth-${toString v.id}";
+          netnsName = "nb-${n}";
+          ipNetns = "${ip} -n ${netnsName}";
+          netnsIptables = "${ip} netns exec ${netnsName} ${config.networking.firewall.package}/bin/iptables";
+        in {
+          "${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}";
+
+          "netns-${n}" = rec {
+            requires = [ "netns-bridge.service" ];
+            after = [ "netns-bridge.service" ];
+            bindsTo = [ "${n}.service" ];
+            requiredBy = bindsTo;
+            before = bindsTo;
+            script = ''
+              ${ip} netns add ${netnsName}
+              ${ipNetns} link set lo up
+              ${ip} link add ${vethName} type veth peer name br-${vethName}
+              ${ip} link set ${vethName} netns ${netnsName}
+              ${ipNetns} addr add ${v.address}/24 dev ${vethName}
+              ${ip} link set br-${vethName} up
+              ${ipNetns} link set ${vethName} up
+              ${ip} link set br-${vethName} master br0
+              ${ipNetns} route add default via ${bridgeIp}
+              ${netnsIptables} -w -P INPUT DROP
+              ${netnsIptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
+            '' + (optionalString (config.services.${n}.enforceTor or false)) ''
+              ${netnsIptables} -w -P OUTPUT DROP
+              ${netnsIptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
+            '' + concatMapStrings (otherNetns: let
+              other = netns.${otherNetns};
+            in ''
+              ${netnsIptables} -w -A INPUT -s ${other.address} -j ACCEPT
+              ${netnsIptables} -w -A OUTPUT -d ${other.address} -j ACCEPT
+            '') v.availableNetns;
+            preStop = ''
+              ${ip} netns delete ${netnsName}
+              ${ip} link del br-${vethName}
+            '';
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = "yes";
+              ExecStartPre = "-${ip} netns delete ${netnsName}";
+            };
+          };
+        };
+      in foldl (services: n:
+        services // (makeNetnsServices n netns.${n})
+      ) {} (builtins.attrNames netns));
+
+      # bitcoin: Custom netns configs
+      services.bitcoind = {
+        bind = netns.bitcoind.address;
+        rpcbind = [
+          "${netns.bitcoind.address}"
+          "127.0.0.1"
+        ];
+        rpcallowip = [
+          "127.0.0.1"
+        ] ++ lib.lists.concatMap (s: [
+          "${netns.${s}.address}"
+        ]) netns.bitcoind.availableNetns;
+        cli = pkgs.writeScriptBin "bitcoin-cli" ''
+          netns-exec nb-bitcoind ${config.services.bitcoind.package}/bin/bitcoin-cli -datadir='${config.services.bitcoind.dataDir}' "$@"
+        '';
+      };
+
+      # clightning: Custom netns configs
+      services.clightning = mkIf config.services.clightning.enable {
+        bitcoin-rpcconnect = netns.bitcoind.address;
+        bind-addr = "${netns.clightning.address}:${toString config.services.clightning.onionport}";
+      };
+
+      # lnd: Custom netns configs
+      services.lnd = mkIf config.services.lnd.enable {
+        listen = netns.lnd.address;
+        rpclisten = [
+          "${netns.lnd.address}"
+          "127.0.0.1"
+        ];
+        restlisten = [
+          "${netns.lnd.address}"
+          "127.0.0.1"
+        ];
+        bitcoind-host = netns.bitcoind.address;
+        cli = pkgs.writeScriptBin "lncli"
+        # Switch user because lnd makes datadir contents readable by user only
+        ''
+          netns-exec nb-lnd sudo -u lnd ${config.services.lnd.package}/bin/lncli --tlscertpath ${config.nix-bitcoin.secretsDir}/lnd-cert \
+            --macaroonpath '${config.services.lnd.dataDir}/chain/bitcoin/mainnet/admin.macaroon' "$@"
+        '';
+      };
+
+      # liquidd: Custom netns configs
+      services.liquidd = mkIf config.services.liquidd.enable {
+        bind = netns.liquidd.address;
+        rpcbind = [
+          "${netns.liquidd.address}"
+          "127.0.0.1"
+        ];
+        rpcallowip = [
+          "127.0.0.1"
+        ] ++ lib.lists.concatMap (s: [
+          "${netns.${s}.address}"
+        ]) netns.liquidd.availableNetns;
+        mainchainrpchost = netns.bitcoind.address;
+        cli = pkgs.writeScriptBin "elements-cli" ''
+          netns-exec nb-liquidd ${pkgs.nix-bitcoin.elementsd}/bin/elements-cli -datadir='${config.services.liquidd.dataDir}' "$@"
+        '';
+        swap-cli = pkgs.writeScriptBin "liquidswap-cli" ''
+          netns-exec nb-liquidd ${pkgs.nix-bitcoin.liquid-swap}/bin/liquidswap-cli -c '${config.services.liquidd.dataDir}/elements.conf' "$@"
+        '';
+      };
+
+      # electrs: Custom netns configs
+      services.electrs = mkIf config.services.electrs.enable {
+        host =  if config.services.electrs.TLSProxy.enable then netns.nginx.address else netns.electrs.address;
+        address = netns.electrs.address;
+        daemonrpc = "${netns.bitcoind.address}:${toString config.services.bitcoind.rpc.port}";
+      };
+
+      # spark-wallet: Custom netns configs
+      services.spark-wallet = mkIf config.services.spark-wallet.enable {
+        host = netns.spark-wallet.address;
+        extraArgs = "--no-tls";
+      };
+
+      # lightning-charge: Custom netns configs
+      services.lightning-charge.host = mkIf config.services.lightning-charge.enable netns.lightning-charge.address;
+
+      # nanopos: Custom netns configs
+      services.nanopos = mkIf config.services.nanopos.enable {
+        charged-url = "http://${netns.lightning-charge.address}:9112";
+        host = netns.nanopos.address;
+      };
+
+      # nginx: Custom netns configs
+      services.nix-bitcoin-webindex.host = mkIf config.services.nix-bitcoin-webindex.enable netns.nginx.address;
+
+    })
+    # Custom netns config option values if netns-isolation not enabled
+    (mkIf (!cfg.enable) {
+      # clightning
+      services.clightning.bind-addr = "127.0.0.1:${toString config.services.clightning.onionport}";
+    })
+  ];
+}

modules/nix-bitcoin-services.nix

@@ -36,7 +36,7 @@ with lib;
   nodejs = { MemoryDenyWriteExecute = "false"; };
   # Allow tor traffic. Allow takes precedence over Deny.
   allowTor = {
-    IPAddressAllow = "127.0.0.1/32 ::1/128";
+    IPAddressAllow = "127.0.0.1/32 ::1/128 169.254.0.0/16";
   };
   # Allow any traffic
   allowAnyIP = { IPAddressAllow = "any"; };

modules/nix-bitcoin-webindex.nix

@@ -13,11 +13,7 @@ let
             nix-bitcoin
           </h1>
         </p>
-        <p>
+        ${optionalString config.services.nanopos.enable ''<p><h2><a href="store/">store</a></h2></p>''}
-        <h2>
-          <a href="store/">store</a>
-        </h2>
-        </p>
         <p>
         <h3>
           lightning node: CLIGHTNING_ID
@@ -43,6 +39,11 @@ in {
         If enabled, the webindex service will be installed.
       '';
     };
+    host = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = "HTTP server listen address.";
+    };
     enforceTor =  nix-bitcoin-services.enforceTor;
   };
 
@@ -61,19 +62,13 @@ in {
       enable = true;
       virtualHosts."_" = {
         root = "/var/www";
-        extraConfig = ''
-          location /store/ {
-            proxy_pass http://127.0.0.1:${toString config.services.nanopos.port};
-            rewrite /store/(.*) /$1 break;
-          }
-        '';
       };
     };
     services.tor.hiddenServices.nginx = {
       map = [{
-        port = 80;
+        port = 80; toHost = cfg.host;
       } {
-        port = 443;
+        port = 443; toHost = cfg.host;
       }];
       version = 3;
     };
@@ -82,7 +77,6 @@ in {
     systemd.services.create-web-index = {
       description = "Get node info";
       wantedBy = [ "multi-user.target" ];
-      after = [ "nodeinfo.service" ];
       path  = with pkgs; [
         config.programs.nodeinfo
         config.services.clightning.cli

modules/presets/secure-node.nix

@@ -24,6 +24,11 @@ in {
       default = 9735;
       description = "Port on which to listen for tor client connections.";
     };
+    services.lnd.onionport = mkOption {
+      type = types.ints.u16;
+      default = 9735;
+      description = "Port on which to listen for tor client connections.";
+    };
     services.electrs.onionport = mkOption {
       type = types.port;
       default = 50002;
@@ -52,6 +57,11 @@ in {
       hiddenServices.sshd = mkHiddenService { port = 22; };
     };
 
+    # netns-isolation
+    nix-bitcoin.netns-isolation = {
+      addressblock = 1;
+    };
+
     # bitcoind
     services.bitcoind = {
       enable = true;
@@ -66,20 +76,22 @@ in {
       addresstype = "bech32";
       dbCache = 1000;
     };
-    services.tor.hiddenServices.bitcoind = mkHiddenService { port = cfg.bitcoind.port; };
+    services.tor.hiddenServices.bitcoind = mkHiddenService { port = cfg.bitcoind.port; toHost = cfg.bitcoind.bind; };
 
     # clightning
     services.clightning = {
-      bitcoin-rpcuser = cfg.bitcoind.rpcuser;
       proxy = cfg.tor.client.socksListenAddress;
       enforceTor = true;
       always-use-proxy = true;
-      bind-addr = "127.0.0.1:${toString cfg.clightning.onionport}";
     };
-    services.tor.hiddenServices.clightning = mkHiddenService { port = cfg.clightning.onionport; };
+    services.tor.hiddenServices.clightning = mkIf cfg.clightning.enable (mkHiddenService { port = cfg.clightning.onionport; toHost = (builtins.head (builtins.split ":" cfg.clightning.bind-addr)); });
 
     # lnd
-    services.lnd.enforceTor = true;
+    services.lnd = {
+      tor-socks = cfg.tor.client.socksListenAddress;
+      enforceTor = true;
+    };
+    services.tor.hiddenServices.lnd = mkIf cfg.lnd.enable (mkHiddenService { port = cfg.lnd.onionport; toHost = cfg.lnd.listen; });
 
     # liquidd
     services.liquidd = {
@@ -95,7 +107,7 @@ in {
       enforceTor = true;
       port = 7042;
     };
-    services.tor.hiddenServices.liquidd = mkHiddenService { port = cfg.liquidd.port; };
+    services.tor.hiddenServices.liquidd = mkIf cfg.liquidd.enable (mkHiddenService { port = cfg.liquidd.port; toHost = cfg.liquidd.bind; });
 
     # electrs
     services.electrs = {
@@ -104,12 +116,22 @@ in {
       TLSProxy.enable = true;
       TLSProxy.port = 50003;
     };
-    services.tor.hiddenServices.electrs = mkHiddenService {
+    services.tor.hiddenServices.electrs = mkIf cfg.electrs.enable (mkHiddenService {
       port = cfg.electrs.onionport;
       toPort = if cfg.electrs.TLSProxy.enable then cfg.electrs.TLSProxy.port else cfg.electrs.port;
+      toHost = cfg.electrs.host;
+    });
+
+    services.spark-wallet = {
+      onion-service = true;
+      enforceTor = true;
     };
 
-    services.spark-wallet.onion-service = true;
+    services.lightning-charge.enforceTor = true;
+
+    services.nanopos.enforceTor = true;
+
+    services.recurring-donations.enforceTor = true;
 
     services.nix-bitcoin-webindex.enforceTor = true;
 

modules/recurring-donations.nix

@@ -11,7 +11,7 @@ let
       NAME=$1
       AMOUNT=$2
       echo Attempting to pay $AMOUNT sat to $NAME
-      INVOICE=$(torsocks curl -d "satoshi_amount=$AMOUNT&payment_method=ln&id=$NAME&type=profile" -X POST https://api.tallyco.in/v1/payment/request/ | jq -r '.lightning_pay_request') 2> /dev/null
+      INVOICE=$(curl --socks5-hostname ${config.services.tor.client.socksListenAddress} -d "satoshi_amount=$AMOUNT&payment_method=ln&id=$NAME&type=profile" -X POST https://api.tallyco.in/v1/payment/request/ | jq -r '.lightning_pay_request') 2> /dev/null
       if [ -z "$INVOICE" ] || [ "$INVOICE" = "null" ]; then
         echo "ERROR: did not get invoice from tallycoin"
         return
@@ -75,6 +75,7 @@ in {
         Random delay to add to scheduled time for donation. Default is one day.
       '';
     };
+    enforceTor =  nix-bitcoin-services.enforceTor;
   };
 
   config = mkIf cfg.enable {
@@ -95,12 +96,14 @@ in {
       description = "Run recurring-donations";
       requires = [ "clightning.service" ];
       after = [ "clightning.service" ];
-      path = with pkgs; [ nix-bitcoin.clightning curl torsocks sudo jq ];
+      path = with pkgs; [ nix-bitcoin.clightning curl sudo jq ];
       serviceConfig = nix-bitcoin-services.defaultHardening // {
         ExecStart = "${pkgs.bash}/bin/bash ${recurring-donations-script}";
         User = "recurring-donations";
         Type = "oneshot";
-      } // nix-bitcoin-services.allowTor;
+      } // (if cfg.enforceTor
+            then nix-bitcoin-services.allowTor
+            else nix-bitcoin-services.allowAnyIP);
     };
     systemd.timers.recurring-donations = {
       requires = [ "clightning.service" ];

modules/spark-wallet.nix

@@ -7,7 +7,7 @@ let
   inherit (config) nix-bitcoin-services;
   onion-chef-service = (if cfg.onion-service then [ "onion-chef.service" ] else []);
   run-spark-wallet = pkgs.writeScript "run-spark-wallet" ''
-    CMD="${pkgs.nix-bitcoin.spark-wallet}/bin/spark-wallet --ln-path ${cfg.ln-path} -Q -k -c ${config.nix-bitcoin.secretsDir}/spark-wallet-login"
+    CMD="${pkgs.nix-bitcoin.spark-wallet}/bin/spark-wallet --ln-path ${cfg.ln-path} --host ${cfg.host} -Q -k -c ${config.nix-bitcoin.secretsDir}/spark-wallet-login ${cfg.extraArgs}"
     ${optionalString cfg.onion-service
       ''
       echo Getting onion hostname
@@ -29,6 +29,11 @@ in {
         If enabled, the spark-wallet service will be installed.
       '';
     };
+    host = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = "http(s) server listen address.";
+    };
     ln-path = mkOption {
       type = types.path;
       default = "${config.services.clightning.dataDir}/bitcoin";
@@ -43,6 +48,12 @@ in {
         "If enabled, configures spark-wallet to be reachable through an onion service.";
       '';
     };
+    extraArgs = mkOption {
+      type = types.separatedString " ";
+      default = "";
+      description = "Extra command line arguments passed to spark-wallet.";
+    };
+    enforceTor =  nix-bitcoin-services.enforceTor;
   };
 
   config = mkIf cfg.enable {
@@ -65,7 +76,7 @@ in {
     services.tor.client.enable = true;
     services.tor.hiddenServices.spark-wallet = mkIf cfg.onion-service {
       map = [{
-        port = 80; toPort = 9737;
+        port = 80; toPort = 9737; toHost = cfg.host;
       }];
       version = 3;
     };
@@ -82,8 +93,10 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = "/var/lib/onion-chef";
-      } // nix-bitcoin-services.nodejs
+      } // (if cfg.enforceTor
-        // nix-bitcoin-services.allowTor;
+            then nix-bitcoin-services.allowTor
+            else nix-bitcoin-services.allowAnyIP)
+        // nix-bitcoin-services.nodejs;
     };
     nix-bitcoin.secrets.spark-wallet-login.user = "spark-wallet";
   };

pkgs/default.nix

@@ -10,6 +10,7 @@
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
   generate-secrets = pkgs.callPackage ./generate-secrets { };
   nixops19_09 = pkgs.callPackage ./nixops { };
+  netns-exec = pkgs.callPackage ./netns-exec { };
 
   pinned = import ./pinned.nix;
 }

pkgs/generate-secrets/openssl.cnf

@@ -30,3 +30,5 @@ subjectAltName			= @alt_names
 [ alt_names ]
 IP.1 = 127.0.0.1
 DNS.1 = localhost
+# TODO: Remove hardcoded lnd IP
+IP.2 = 169.254.1.14

pkgs/netns-exec/default.nix

@@ -0,0 +1,11 @@
+{ stdenv, pkgs }:
+
+stdenv.mkDerivation {
+  name = "netns-exec";
+  buildInputs = [ pkgs.libcap ];
+  src = ./src;
+  installPhase = ''
+    mkdir -p $out
+    cp main $out/netns-exec
+  '';
+}

pkgs/netns-exec/src/Makefile

@@ -0,0 +1,2 @@
+main: main.c
+	gcc -lcap -o main main.c

pkgs/netns-exec/src/main.c

@@ -0,0 +1,86 @@
+/* This program must be run with CAP_SYS_ADMIN. This can be achieved for example
+ * with
+ *         # setcap CAP_SYS_ADMIN+ep ./main
+ */
+
+#define _GNU_SOURCE
+#include <sched.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/capability.h>
+
+static char *available_netns[] = {
+    "nb-lnd",
+    "nb-bitcoind",
+    "nb-liquidd"
+};
+
+int check_netns(char *netns) {
+    int i;
+    int n_available_netns = sizeof(available_netns) / sizeof(available_netns[0]);
+    for (i = 0; i < n_available_netns; i++) {
+        if (strcmp(available_netns[i], netns) == 0) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+void print_capabilities() {
+    cap_t caps = cap_get_proc();
+    printf("Capabilities: %s\n", cap_to_text(caps, NULL));
+    cap_free(caps);
+}
+void drop_capabilities() {
+    cap_t caps = cap_get_proc();
+    cap_clear(caps);
+    cap_set_proc(caps);
+    cap_free(caps);
+}
+
+int main(int argc, char **argv) {
+    int fd;
+    char netns_path[256];
+
+    if (argc < 3) {
+        printf("usage: %s <netns> <command to execute>\n", argv[0]);
+        return 1;
+    }
+
+    if (!check_netns(argv[1])) {
+        printf("Failed checking %s against available netns.\n", argv[1]);
+        return 1;
+    }
+
+    if(snprintf(netns_path, sizeof(netns_path), "/var/run/netns/%s", argv[1]) < 0) {
+        printf("Failed concatenating %s to the netns path.\n", argv[1]);
+        return 1;
+    }
+
+    fd = open(netns_path, O_RDONLY);
+    if (fd < 0) {
+        printf("Failed opening netns %s: %d, %s \n", netns_path, errno, strerror(errno));
+        return 1;
+    }
+
+    if (setns(fd, CLONE_NEWNET) < 0) {
+        printf("Failed setns %d, %s \n", errno, strerror(errno));
+        return 1;
+    }
+
+    /* Drop capabilities */
+    #ifdef DEBUG
+    print_capabilities();
+    #endif
+    drop_capabilities();
+    #ifdef DEBUG
+    print_capabilities();
+    #endif
+
+    execvp(argv[2], &argv[2]);
+    return 0;
+}
+

test/run-tests.sh

@@ -5,19 +5,52 @@
 #
 # Usage:
 #   Run test
-#   ./run-tests.sh
+#   ./run-tests.sh --scenario <scenario>
 #
 #   Run test and save result to avoid garbage collection
-#   ./run-tests.sh build --out-link /tmp/nix-bitcoin-test
+#   ./run-tests.sh --scenario <scenario> build --out-link /tmp/nix-bitcoin-test
 #
 #   Run interactive test debugging
-#   ./run-tests.sh debug
+#   ./run-tests.sh --scenario <scenario> debug
 #
 #   This starts the testing VM and drops you into a Python REPL where you can
 #   manually execute the tests from ./test-script.py
 
 set -eo pipefail
 
+die() {
+    printf '%s\n' "$1" >&2
+    exit 1
+}
+
+# Initialize all the option variables.
+# This ensures we are not contaminated by variables from the environment.
+scenario=
+
+while :; do
+    case $1 in
+	--scenario)
+	    if [ "$2" ]; then
+		scenario=$2
+		shift
+	    else
+		die 'ERROR: "--scenario" requires a non-empty option argument.'
+	    fi
+	    ;;
+	-?*)
+	    printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2
+	    ;;
+	*)
+	    break
+    esac
+
+    shift
+done
+
+if [[ -z $scenario ]]; then
+    die 'ERROR: "--scenario" is required'
+fi
+
 numCPUs=${numCPUs:-$(nproc)}
 # Min. 800 MiB needed to avoid 'out of memory' errors
 memoryMiB=${memoryMiB:-2048}
@@ -32,7 +65,7 @@ run() {
     export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX)
     trap "rm -rf $TMPDIR" EXIT
 
-    nix-build --out-link $TMPDIR/driver "$scriptDir/test.nix" -A driver
+    nix-build --out-link $TMPDIR/driver -E "import \"$scriptDir/test.nix\" { scenario = \"$scenario\"; }" -A driver
 
     # Variable 'tests' contains the Python code that is executed by the driver on startup
     if [[ $1 == --interactive ]]; then
@@ -95,7 +128,7 @@ exprForCI() {
 
 vmTestNixExpr() {
   cat <<EOF
-    (import "$scriptDir/test.nix" {}).overrideAttrs (old: rec {
+    (import "$scriptDir/test.nix" { scenario = "$scenario"; } {}).overrideAttrs (old: rec {
       buildCommand = ''
         export QEMU_OPTS="-smp $numCPUs -m $memoryMiB"
         echo "VM stats: CPUs: $numCPUs, memory: $memoryMiB MiB"

test/scenarios/default.py

@@ -1,32 +1,3 @@
-def succeed(*cmds):
-    """Returns the concatenated output of all cmds"""
-    return machine.succeed(*cmds)
-
-
-def assert_matches(cmd, regexp):
-    out = succeed(cmd)
-    if not re.search(regexp, out):
-        raise Exception(f"Pattern '{regexp}' not found in '{out}'")
-
-
-def log_has_string(unit, str):
-    return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
-
-
-def assert_no_failure(unit):
-    """Unit should not have failed since the system is running"""
-    machine.fail(log_has_string(unit, "Failed with result"))
-
-
-def assert_running(unit):
-    machine.wait_for_unit(unit)
-    assert_no_failure(unit)
-
-
-# Don't execute the following test suite when this script is running in interactive mode
-if "is_interactive" in vars():
-    raise Exception()
-
 ### Tests
 
 assert_running("setup-secrets")

test/scenarios/lib.py

@@ -0,0 +1,34 @@
+def succeed(*cmds):
+    """Returns the concatenated output of all cmds"""
+    return machine.succeed(*cmds)
+
+
+def assert_matches(cmd, regexp):
+    out = succeed(cmd)
+    if not re.search(regexp, out):
+        raise Exception(f"Pattern '{regexp}' not found in '{out}'")
+
+
+def assert_matches_exactly(cmd, regexp):
+    out = succeed(cmd)
+    if not re.fullmatch(regexp, out):
+        raise Exception(f"Pattern '{regexp}' doesn't match '{out}'")
+
+
+def log_has_string(unit, str):
+    return f"journalctl -b --output=cat -u {unit} --grep='{str}'"
+
+
+def assert_no_failure(unit):
+    """Unit should not have failed since the system is running"""
+    machine.fail(log_has_string(unit, "Failed with result"))
+
+
+def assert_running(unit):
+    machine.wait_for_unit(unit)
+    assert_no_failure(unit)
+
+
+# Don't execute the following test suite when this script is running in interactive mode
+if "is_interactive" in vars():
+    raise Exception()

test/scenarios/withnetns.py

@@ -0,0 +1,139 @@
+# netns IP addresses
+bitcoind_ip = "169.254.1.12"
+clightning_ip = "169.254.1.13"
+lnd_ip = "169.254.1.14"
+liquidd_ip = "169.254.1.15"
+electrs_ip = "169.254.1.16"
+sparkwallet_ip = "169.254.1.17"
+lightningcharge_ip = "169.254.1.18"
+nanopos_ip = "169.254.1.19"
+recurringdonations_ip = "169.254.1.20"
+nginx_ip = "169.254.1.21"
+
+### Tests
+
+assert_running("setup-secrets")
+# Unused secrets should be inaccessible
+succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]')
+
+assert_running("bitcoind")
+machine.wait_until_succeeds("bitcoin-cli getnetworkinfo")
+assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"')
+
+assert_running("electrs")
+machine.wait_until_succeeds(
+    "ip netns exec nb-electrs nc -z localhost 4224"
+)  # prometeus metrics provider
+# Check RPC connection to bitcoind
+machine.wait_until_succeeds(log_has_string("electrs", "NetworkInfo"))
+assert_running("nginx")
+# SSL stratum server via nginx. Only check for open port, no content is served here
+# as electrs isn't ready.
+machine.wait_until_succeeds("ip netns exec nb-nginx nc -z localhost 50003")
+# Stop electrs from spamming the test log with 'wait for bitcoind sync' messages
+succeed("systemctl stop electrs")
+
+assert_running("liquidd")
+machine.wait_until_succeeds("elements-cli getnetworkinfo")
+assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"')
+succeed("su operator -c 'liquidswap-cli --help'")
+
+assert_running("clightning")
+assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+
+assert_running("spark-wallet")
+spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1]
+machine.wait_until_succeeds("ip netns exec nb-spark-wallet nc -z %s 9737" % sparkwallet_ip)
+assert_matches(
+    f"ip netns exec nb-spark-wallet curl -s {spark_auth}@%s:9737" % sparkwallet_ip, "Spark"
+)
+
+assert_running("lightning-charge")
+charge_auth = re.search("API_TOKEN=(.*)", succeed("cat /secrets/lightning-charge-env"))[1]
+machine.wait_until_succeeds("ip netns exec nb-nanopos nc -z %s 9112" % lightningcharge_ip)
+assert_matches(
+    f"ip netns exec nb-nanopos curl -s api-token:{charge_auth}@%s:9112/info | jq"
+    % lightningcharge_ip,
+    '"id"',
+)
+
+assert_running("nanopos")
+machine.wait_until_succeeds("ip netns exec nb-lightning-charge nc -z %s 9116" % nanopos_ip)
+assert_matches("ip netns exec nb-lightning-charge curl %s:9116" % nanopos_ip, "tshirt")
+
+assert_running("onion-chef")
+
+# FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due
+# to incomplete unit dependencies.
+# 'create-web-index' implicitly tests 'nodeinfo'.
+machine.wait_for_unit("create-web-index")
+machine.wait_until_succeeds("ip netns exec nb-nginx nc -z localhost 80")
+assert_matches("ip netns exec nb-nginx curl localhost", "nix-bitcoin")
+assert_matches("ip netns exec nb-nginx curl -L localhost/store", "tshirt")
+
+machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist"))
+assert_no_failure("bitcoind-import-banlist")
+
+### Security tests
+
+ping_bitcoind = "ip netns exec nb-bitcoind ping -c 1 -w 1"
+ping_nanopos = "ip netns exec nb-nanopos ping -c 1 -w 1"
+
+# Positive ping tests (non-exhaustive)
+machine.succeed(
+    "%s %s &&" % (ping_bitcoind, bitcoind_ip)
+    + "%s %s &&" % (ping_bitcoind, clightning_ip)
+    + "%s %s &&" % (ping_bitcoind, liquidd_ip)
+    + "%s %s &&" % (ping_nanopos, lightningcharge_ip)
+    + "%s %s &&" % (ping_nanopos, nanopos_ip)
+    + "%s %s" % (ping_nanopos, nginx_ip)
+)
+
+# Negative ping tests (non-exhaustive)
+machine.fail(
+    "%s %s ||" % (ping_bitcoind, sparkwallet_ip)
+    + "%s %s ||" % (ping_bitcoind, lightningcharge_ip)
+    + "%s %s ||" % (ping_bitcoind, nanopos_ip)
+    + "%s %s ||" % (ping_bitcoind, recurringdonations_ip)
+    + "%s %s ||" % (ping_bitcoind, nginx_ip)
+    + "%s %s ||" % (ping_nanopos, bitcoind_ip)
+    + "%s %s ||" % (ping_nanopos, clightning_ip)
+    + "%s %s ||" % (ping_nanopos, lnd_ip)
+    + "%s %s ||" % (ping_nanopos, liquidd_ip)
+    + "%s %s ||" % (ping_nanopos, electrs_ip)
+    + "%s %s ||" % (ping_nanopos, sparkwallet_ip)
+    + "%s %s" % (ping_nanopos, recurringdonations_ip)
+)
+
+# test that netns-exec can't be run for unauthorized namespace
+machine.fail("netns-exec nb-electrs ip a")
+
+# test that netns-exec drops capabilities
+assert_matches_exactly(
+    "su operator -c 'netns-exec nb-bitcoind capsh --print | grep Current '", "Current: =\n"
+)
+
+# test that netns-exec can not be executed by users that are not operator
+machine.fail("sudo -u clightning netns-exec nb-bitcoind ip a")
+
+### Additional tests
+
+# Current time in µs
+pre_restart = succeed("date +%s.%6N").rstrip()
+
+# Sanity-check system by restarting all services
+succeed("systemctl restart bitcoind clightning spark-wallet lightning-charge nanopos liquidd")
+
+# Now that the bitcoind restart triggered a banlist import restart, check that
+# re-importing already banned addresses works
+machine.wait_until_succeeds(
+    log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist")
+)
+assert_no_failure("bitcoind-import-banlist")
+
+### Test lnd
+
+succeed("systemctl stop nanopos lightning-charge spark-wallet clightning")
+succeed("systemctl start lnd")
+assert_matches("su operator -c 'lncli getinfo' | jq", '"version"')
+assert_no_failure("lnd")

test/test.nix

@@ -1,5 +1,12 @@
 # Integration test, can be run without internet access.
 
+{ scenario ? "default" }:
+
+let
+  netns-isolation = builtins.getAttr scenario { default = false; withnetns = true; };
+  testScriptFilename = builtins.getAttr scenario { default = ./scenarios/default.py; withnetns = ./scenarios/withnetns.py; };
+in
+
 import ./make-test.nix rec {
   name = "nix-bitcoin";
 
@@ -16,6 +23,8 @@ import ./make-test.nix rec {
       # hardened
     ];
 
+    nix-bitcoin.netns-isolation.enable = mkForce netns-isolation;
+
     services.bitcoind.extraConfig = mkForce "connect=0";
 
     services.clightning.enable = true;
@@ -46,6 +55,5 @@ import ./make-test.nix rec {
       install -o nobody -g nogroup -m777 <(:) /secrets/dummy
     '';
   };
-
+  testScript = builtins.readFile ./scenarios/lib.py + "\n\n" + builtins.readFile testScriptFilename;
-  testScript = builtins.readFile ./test-script.py;
 }