nix-bitcoin/modules/netns-isolation.nix

354 lines
11 KiB
Nix
Raw Normal View History

{ config, lib, pkgs, ... }:
with lib;
let
options.nix-bitcoin.netns-isolation = {
enable = mkEnableOption "netns isolation";
addressblock = mkOption {
type = types.ints.u8;
default = 1;
description = ''
The address block N in 169.254.N.0/24, used as the prefix for netns addresses.
'';
};
services = mkOption {
default = {};
type = types.attrsOf (types.submodule {
options = {
id = mkOption {
# TODO: Assert uniqueness
2020-08-21 13:35:54 -07:00
type = types.ints.between 11 255;
description = ''
2020-08-21 13:35:54 -07:00
id for the netns, used for the IP address host part and
for naming the interfaces. Must be unique. Must be greater than 10.
'';
};
connections = mkOption {
type = with types; listOf str;
default = [];
};
};
});
};
allowedUser = mkOption {
type = types.str;
description = ''
User that is allowed to execute commands in the service network namespaces.
The user's group is also authorized.
'';
2020-09-28 04:09:03 -07:00
default = config.nix-bitcoin.operator.name;
};
netns = mkOption {
readOnly = true;
default = netns;
description = "Exposes netns parameters.";
};
bridgeIp = mkOption {
readOnly = true;
default = bridgeIp;
description = "IP of the netns bridge interface.";
};
};
cfg = config.nix-bitcoin.netns-isolation;
netns = builtins.mapAttrs (n: v: {
inherit (v) id;
address = "169.254.${toString cfg.addressblock}.${toString v.id}";
availableNetns = availableNetns.${n};
netnsName = "nb-${n}";
}) enabledServices;
# Symmetric netns connection matrix
# if clightning.connections = [ "bitcoind" ]; then
# availableNetns.bitcoind = [ "clighting" ];
# and
# availableNetns.clighting = [ "bitcoind" ];
#
# FIXME: Although negligible for our purposes, this calculation's runtime
# is in the order of (number of connections * number of services),
# because attrsets and lists are fully copied on each update with '//' or '++'.
# This can only be improved with an update in the nix language.
#
availableNetns = let
# base = { clightning = [ "bitcoind" ]; ... }
base = builtins.mapAttrs (n: v:
builtins.filter isEnabled v.connections
) enabledServices;
in
foldl (xs: s1:
foldl (xs: s2:
xs // { "${s2}" = xs.${s2} ++ [ s1 ]; }
) xs cfg.services.${s1}.connections
) base (builtins.attrNames base);
enabledServices = filterAttrs (n: v: isEnabled n) cfg.services;
isEnabled = x: config.services.${x}.enable;
ip = "${pkgs.iproute}/bin/ip";
iptables = "${config.networking.firewall.package}/bin/iptables";
bridgeIp = "169.254.${toString cfg.addressblock}.10";
mkCliExec = service: "exec netns-exec ${netns.${service}.netnsName}";
in {
inherit options;
config = mkIf cfg.enable (mkMerge [
# Base infrastructure
{
networking.dhcpcd.denyInterfaces = [ "nb-br" "nb-veth*" ];
2021-08-04 15:49:00 -07:00
services.tor.client.socksListenAddress = {
addr = bridgeIp;
# Default NixOS values. These must be repeated when redefining this option.
port = 9050;
IsolateDestAddr = true;
};
2021-10-10 06:14:38 -07:00
services.i2pd.proto.sam.address = bridgeIp;
networking.firewall.interfaces.nb-br.allowedTCPPorts = [
config.services.tor.client.socksListenAddress.port
config.services.i2pd.proto.sam.port
];
boot.kernel.sysctl."net.ipv4.ip_forward" = true;
security.wrappers.netns-exec = {
source = config.nix-bitcoin.pkgs.netns-exec;
capabilities = "cap_sys_admin=ep";
owner = cfg.allowedUser;
permissions = "550";
};
systemd.services = {
# Due to a NixOS bug we can't currently use option `networking.bridges` to
# setup the bridge while `networking.useDHCP` is enabled.
nb-netns-bridge = {
description = "nix-bitcoin netns bridge";
wantedBy = [ "network-setup.service" ];
partOf = [ "network-setup.service" ];
before = [ "network-setup.service" ];
after = [ "network-pre.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
${ip} link add name nb-br type bridge
${ip} link set nb-br up
${ip} addr add ${bridgeIp}/24 brd + dev nb-br
${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
'';
preStop = ''
${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE
${ip} link del nb-br
'';
};
2020-04-23 09:18:47 -07:00
} //
(let
makeNetnsServices = n: v: let
veth = "nb-veth-${toString v.id}";
peer = "nb-veth-br-${toString v.id}";
inherit (v) netnsName;
nsenter = "${pkgs.utillinux}/bin/nsenter";
2020-10-29 13:20:41 -07:00
allowedAddresses = concatMapStringsSep "," (available: netns.${available}.address) v.availableNetns;
setup = ''
${ip} netns add ${netnsName}
${ip} link add ${veth} type veth peer name ${peer}
${ip} link set ${veth} netns ${netnsName}
# The peer link is never used directly, so don't auto-assign an IPv6 address
echo 1 > /proc/sys/net/ipv6/conf/${peer}/disable_ipv6
${ip} link set ${peer} up
${ip} link set ${peer} master nb-br
exec ${nsenter} --net=/run/netns/${netnsName} ${script "in-netns" setupInNetns}
'';
setupInNetns = ''
${ip} link set lo up
${ip} addr add ${v.address}/24 dev ${veth}
${ip} link set ${veth} up
${ip} route add default via ${bridgeIp}
${iptables} -w -P INPUT DROP
${iptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
# allow return traffic to outgoing connections initiated by the service itself
${iptables} -w -A INPUT -m conntrack --ctstate ESTABLISHED -j ACCEPT
'' + optionalString (config.services.${n}.enforceTor or false) ''
${iptables} -w -P OUTPUT DROP
${iptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT
'' + optionalString (v.availableNetns != []) ''
${iptables} -w -A INPUT -s ${allowedAddresses} -j ACCEPT
${iptables} -w -A OUTPUT -d ${allowedAddresses} -j ACCEPT
'';
script = name: src: pkgs.writers.writeDash name ''
set -e
${src}
'';
in {
"${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}";
"netns-${n}" = rec {
requires = [ "nb-netns-bridge.service" ];
after = [ "nb-netns-bridge.service" ];
requiredBy = [ "${n}.service" ];
before = requiredBy;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = script "setup" setup;
};
2020-11-03 12:54:14 -08:00
# Link deletion is implicit in netns deletion, but it sometimes only happens
# after `netns delete` finishes. Add an extra `link del` to ensure that
# the link is deleted before the service stops, which is needed for service
# restart to succeed.
preStop = ''
${ip} netns delete ${netnsName}
2020-11-03 12:54:14 -08:00
${ip} link del ${peer} 2> /dev/null || true
'';
};
};
in foldl (services: n:
services // (makeNetnsServices n netns.${n})
) {} (builtins.attrNames netns));
}
# Service-specific config
{
nix-bitcoin.netns-isolation.services = {
bitcoind = {
id = 12;
};
clightning = {
id = 13;
connections = [ "bitcoind" ];
};
lnd = {
id = 14;
connections = [ "bitcoind" ];
};
liquidd = {
id = 15;
connections = [ "bitcoind" ];
};
electrs = {
id = 16;
connections = [ "bitcoind" ];
};
spark-wallet = {
id = 17;
# communicates with clightning over lightning-rpc socket
};
recurring-donations = {
id = 20;
# communicates with clightning over lightning-rpc socket
};
nginx = {
id = 21;
};
lightning-loop = {
id = 22;
connections = [ "lnd" ];
};
2020-08-12 07:47:56 -07:00
nbxplorer = {
id = 23;
2021-07-13 07:03:08 -07:00
connections = [ "bitcoind" ]
++ optional config.services.btcpayserver.lbtc "liquidd";
2020-08-12 07:47:56 -07:00
};
btcpayserver = {
id = 24;
connections = [ "nbxplorer" ]
2021-07-13 07:03:08 -07:00
++ optional (config.services.btcpayserver.lightningBackend == "lnd") "lnd"
++ optional config.services.btcpayserver.lbtc "liquidd";
2020-08-12 07:47:56 -07:00
# communicates with clightning over rpc socket
};
2020-04-23 09:18:47 -07:00
joinmarket = {
id = 25;
connections = [ "bitcoind" ];
};
2021-01-17 04:24:57 -08:00
joinmarket-ob-watcher = {
id = 26;
connections = [ "bitcoind" ];
2021-01-17 04:24:57 -08:00
};
2021-03-01 01:59:23 -08:00
lightning-pool = {
id = 27;
connections = [ "lnd" ];
};
2021-06-01 18:11:26 -07:00
charge-lnd = {
id = 28;
connections = [ "lnd" "electrs" ];
};
2021-11-08 03:43:14 -08:00
rtl = {
id = 29;
connections = []
++ optional (config.services.rtl.nodes.lnd) "lnd"
++ optional config.services.rtl.loop "lightning-loop";
};
};
services.bitcoind = {
address = netns.bitcoind.address;
rpc.address = netns.bitcoind.address;
rpc.allowip = [
bridgeIp # For operator user
netns.bitcoind.address
] ++ map (n: netns.${n}.address) netns.bitcoind.availableNetns;
};
systemd.services.bitcoind-import-banlist.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-bitcoind";
services.clightning.address = netns.clightning.address;
services.lnd = {
address = netns.lnd.address;
rpcAddress = netns.lnd.address;
restAddress = netns.lnd.address;
};
services.liquidd = {
address = netns.liquidd.address;
rpc.address = netns.liquidd.address;
rpcallowip = [
bridgeIp # For operator user
netns.liquidd.address
] ++ map (n: netns.${n}.address) netns.liquidd.availableNetns;
};
services.electrs.address = netns.electrs.address;
services.spark-wallet = {
address = netns.spark-wallet.address;
extraArgs = "--no-tls";
};
services.lightning-loop.rpcAddress = netns.lightning-loop.address;
2020-08-12 07:47:56 -07:00
services.nbxplorer.address = netns.nbxplorer.address;
services.btcpayserver.address = netns.btcpayserver.address;
2020-04-23 09:18:47 -07:00
2021-10-24 12:14:09 -07:00
services.joinmarket = {
payjoinAddress = netns.joinmarket.address;
cliExec = mkCliExec "joinmarket";
};
2020-04-23 09:18:47 -07:00
systemd.services.joinmarket-yieldgenerator.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-joinmarket";
2021-01-17 04:24:57 -08:00
services.joinmarket-ob-watcher.address = netns.joinmarket-ob-watcher.address;
2021-03-01 01:59:23 -08:00
services.lightning-pool.rpcAddress = netns.lightning-pool.address;
2021-11-08 03:43:14 -08:00
services.rtl.address = netns.rtl.address;
systemd.services.cl-rest = {
serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-rtl";
requires = [ "netns-rtl.service" ] ;
after = [ "netns-rtl.service" ];
};
}
]);
}