Merge #200: backups: add module

22c3fd52e1 backups: add feature test (nixbitcoin)
e4fb7a52de backups: add module (nixbitcoin)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 22c3fd52e1

Tree-SHA512: 625c1fe4f12ea881b5adb04e07187eae60451402462cd3032b2f741b3f23ee73ea68b98aeb8cfd9206890e8227229cb4ab0cdb5f7935f34fc33fc50dc5df26c9
This commit is contained in:
Jonas Nick 2020-08-04 15:38:38 +00:00
commit 72000b4a99
No known key found for this signature in database
GPG Key ID: 4861DBF262123605
9 changed files with 147 additions and 5 deletions

View File

@ -137,6 +137,27 @@
# interact with off/on chain bridge using `loop in` and `loop out`. # interact with off/on chain bridge using `loop in` and `loop out`.
# services.lightning-loop.enable = true; # services.lightning-loop.enable = true;
### Backups
# Enable this module to use nix-bitcoin's own backups module. By default, it
# uses duplicity to incrementally back up all important files in /var/lib to
# /var/lib/localBackups once a day.
# services.backups.enable = true;
# You can pull the localBackups folder with
# `nixops scp --from bitcoin-node /var/lib/localBackups /my-backup-path/`
# Alternatively, you can also set a remote target url, for example
# services.backups.destination = "sftp://user@host[:port]/[relative|/absolute]_path";
# Supply the sftp password by appending the FTP_PASSWORD environment variable
# to secrets/backup-encryption-env like so
# `echo "FTP_PASSWORD=<password>" >> secrets/backup-encryption-env`
# You many also need to set a ssh host and publickey with
# programs.ssh.knownHosts."host" = {
# hostNames = [ "host" ];
# publicKey = "<ssh public from `ssh-keyscan`>";
# };
# If you also want to backup bulk data like the Bitcoin & Liquid blockchains
# and electrs data directory, enable
# services.backups.with-bulk-data = true;
# FIXME: Define your hostname. # FIXME: Define your hostname.
networking.hostName = "nix-bitcoin"; networking.hostName = "nix-bitcoin";
time.timeZone = "UTC"; time.timeZone = "UTC";

87
modules/backups.nix Normal file
View File

@ -0,0 +1,87 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.backups;
filelist = pkgs.writeText "filelist.txt" ''
${optionalString (!cfg.with-bulk-data) "- ${config.services.bitcoind.dataDir}/blocks"}
${optionalString (!cfg.with-bulk-data) "- ${config.services.bitcoind.dataDir}/chainstate"}
${config.services.bitcoind.dataDir}
${config.services.clightning.dataDir}
${config.services.lnd.dataDir}
/secrets/lnd-seed-mnemonic
${optionalString (!cfg.with-bulk-data) "- ${config.services.liquidd.dataDir}/*/blocks"}
${optionalString (!cfg.with-bulk-data) "- ${config.services.liquidd.dataDir}/*/chainstate"}
${config.services.liquidd.dataDir}
${optionalString cfg.with-bulk-data "${config.services.electrs.dataDir}"}
${config.services.lightning-charge.dataDir}
/var/lib/tor
# Extra files
${cfg.extraFiles}
# Exclude all unspecified files and directories
- /
'';
in {
options.services.backups = {
enable = mkEnableOption "Backups service";
program = mkOption {
type = types.enum [ "duplicity" ];
default = "duplicity";
description = ''
Program with which to do backups.
'';
};
with-bulk-data = mkOption {
type = types.bool;
default = false;
description = ''
Whether to also backup Bitcoin blockchain and other bulk data.
'';
};
destination = mkOption {
type = types.str;
default = "file:///var/lib/localBackups";
description = ''
Where to back up to.
'';
};
frequency = mkOption {
type = types.nullOr types.str;
default = null;
description = ''
Run backup with the given frequency. If null, do not run automatically.
'';
};
extraFiles = mkOption {
type = types.lines;
default = "";
example = ''
/var/lib/nginx
'';
description = "Additional files to be appended to filelist.";
};
};
config = mkMerge [
(mkIf (cfg.enable && cfg.program == "duplicity") {
environment.systemPackages = [ pkgs.duplicity ];
services.duplicity = {
enable = true;
extraFlags = [
"--include-filelist" "${filelist}"
"--full-if-older-than" "1M"
];
targetUrl = "${cfg.destination}";
frequency = cfg.frequency;
secretFile = "${config.nix-bitcoin.secretsDir}/backup-encryption-env";
};
nix-bitcoin.secrets.backup-encryption-env.user = "root";
})
];
}

View File

@ -17,6 +17,7 @@
./secrets/secrets.nix ./secrets/secrets.nix
./netns-isolation.nix ./netns-isolation.nix
./dbus.nix ./dbus.nix
./backups.nix
]; ];
disabledModules = [ "services/networking/bitcoind.nix" ]; disabledModules = [ "services/networking/bitcoind.nix" ];

View File

@ -210,6 +210,11 @@ in {
services.nix-bitcoin-webindex.enforceTor = true; services.nix-bitcoin-webindex.enforceTor = true;
# Backups
services.backups = {
program = "duplicity";
frequency = "daily";
};
environment.systemPackages = with pkgs; [ environment.systemPackages = with pkgs; [
tor tor

View File

@ -12,12 +12,14 @@ makePasswordSecret lnd-wallet-password
makePasswordSecret liquid-rpcpassword makePasswordSecret liquid-rpcpassword
makePasswordSecret lightning-charge-token makePasswordSecret lightning-charge-token
makePasswordSecret spark-wallet-password makePasswordSecret spark-wallet-password
makePasswordSecret backup-encryption-password
[[ -e bitcoin-HMAC-privileged ]] || rpcauth privileged $(cat bitcoin-rpcpassword-privileged) | grep rpcauth | cut -d ':' -f 2 > bitcoin-HMAC-privileged [[ -e bitcoin-HMAC-privileged ]] || rpcauth privileged $(cat bitcoin-rpcpassword-privileged) | grep rpcauth | cut -d ':' -f 2 > bitcoin-HMAC-privileged
[[ -e bitcoin-HMAC-public ]] || rpcauth public $(cat bitcoin-rpcpassword-public) | grep rpcauth | cut -d ':' -f 2 > bitcoin-HMAC-public [[ -e bitcoin-HMAC-public ]] || rpcauth public $(cat bitcoin-rpcpassword-public) | grep rpcauth | cut -d ':' -f 2 > bitcoin-HMAC-public
[[ -e lightning-charge-env ]] || echo "API_TOKEN=$(cat lightning-charge-token)" > lightning-charge-env [[ -e lightning-charge-env ]] || echo "API_TOKEN=$(cat lightning-charge-token)" > lightning-charge-env
[[ -e nanopos-env ]] || echo "CHARGE_TOKEN=$(cat lightning-charge-token)" > nanopos-env [[ -e nanopos-env ]] || echo "CHARGE_TOKEN=$(cat lightning-charge-token)" > nanopos-env
[[ -e spark-wallet-login ]] || echo "login=spark-wallet:$(cat spark-wallet-password)" > spark-wallet-login [[ -e spark-wallet-login ]] || echo "login=spark-wallet:$(cat spark-wallet-password)" > spark-wallet-login
[[ -e backup-encryption-env ]] || echo "PASSPHRASE=$(cat backup-encryption-password)" > backup-encryption-env
if [[ ! -e lnd-key || ! -e lnd-cert ]]; then if [[ ! -e lnd-key || ! -e lnd-cert ]]; then
openssl ecparam -genkey -name prime256v1 -out lnd-key openssl ecparam -genkey -name prime256v1 -out lnd-key

View File

@ -25,7 +25,7 @@ def web_index():
assert_matches("curl -L localhost/store", "tshirt") assert_matches("curl -L localhost/store", "tshirt")
def final(): def prestop():
pass pass
@ -35,7 +35,7 @@ extra_tests = {
"lightning-charge": lightning_charge, "lightning-charge": lightning_charge,
"nanopos": nanopos, "nanopos": nanopos,
"web-index": web_index, "web-index": web_index,
"final": final, "prestop": prestop,
} }
run_tests(extra_tests) run_tests(extra_tests)

View File

@ -127,7 +127,31 @@ def run_tests(extra_tests):
) )
assert_no_failure("bitcoind-import-banlist") assert_no_failure("bitcoind-import-banlist")
extra_tests.pop("final")() extra_tests.pop("prestop")()
### Test duplicity
succeed("systemctl stop bitcoind")
succeed("systemctl start duplicity")
machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded."))
# Make sure files in duplicity backup and /var/lib are identical
assert_matches(
"export $(cat /secrets/backup-encryption-env); duplicity verify '--archive-dir' '/var/lib/duplicity' 'file:///var/lib/localBackups' '/var/lib'",
"0 differences found",
)
# Make sure duplicity backup includes important files
assert_matches(
"export $(cat /secrets/backup-encryption-env); duplicity list-current-files 'file:///var/lib/localBackups'",
"var/lib/clightning/bitcoin/hsm_secret",
)
assert_matches(
"export $(cat /secrets/backup-encryption-env); duplicity list-current-files 'file:///var/lib/localBackups'",
"secrets/lnd-seed-mnemonic",
)
assert_matches(
"export $(cat /secrets/backup-encryption-env); duplicity list-current-files 'file:///var/lib/localBackups'",
"var/lib/bitcoind/wallet.dat",
)
### Check that all extra_tests have been run ### Check that all extra_tests have been run
assert len(extra_tests) == 0 assert len(extra_tests) == 0

View File

@ -47,7 +47,7 @@ def web_index():
assert_matches("ip netns exec nb-nginx curl -L localhost/store", "tshirt") assert_matches("ip netns exec nb-nginx curl -L localhost/store", "tshirt")
def final(): def prestop():
ping_bitcoind = "ip netns exec nb-bitcoind ping -c 1 -w 1" ping_bitcoind = "ip netns exec nb-bitcoind ping -c 1 -w 1"
ping_nanopos = "ip netns exec nb-nanopos ping -c 1 -w 1" ping_nanopos = "ip netns exec nb-nanopos ping -c 1 -w 1"
@ -98,7 +98,7 @@ extra_tests = {
"lightning-charge": lightning_charge, "lightning-charge": lightning_charge,
"nanopos": nanopos, "nanopos": nanopos,
"web-index": web_index, "web-index": web_index,
"final": final, "prestop": prestop,
} }
run_tests(extra_tests) run_tests(extra_tests)

View File

@ -54,6 +54,8 @@ import ./make-test.nix rec {
ledger = true; ledger = true;
}; };
services.backups.enable = true;
# to test that unused secrets are made inaccessible by 'setup-secrets' # to test that unused secrets are made inaccessible by 'setup-secrets'
systemd.services.generate-secrets.postStart = '' systemd.services.generate-secrets.postStart = ''
install -o nobody -g nogroup -m777 <(:) /secrets/dummy install -o nobody -g nogroup -m777 <(:) /secrets/dummy