From de77281cba44fe4201770978810803f450ffd40d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 16 Aug 2021 10:42:06 +0200 Subject: [PATCH 1/4] pkgs: import pinned nixpkgs in default.nix pkgs/default.nix now explicitly specifies all its dependencies as arguments. This is required for flake support. Also simplify pinned.nix and python-packages by removing unused attrs. --- pkgs/default.nix | 10 ++++++++-- pkgs/pinned.nix | 15 +++------------ pkgs/python-packages/default.nix | 2 -- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pkgs/default.nix b/pkgs/default.nix index 4374aae..039cd08 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -1,4 +1,10 @@ -{ pkgs ? import {} }: +let + nixpkgsPinned = import ./nixpkgs-pinned.nix; +in +# Set default values for use without flakes +{ pkgs ? import { config = {}; overlays = []; } +, pkgsUnstable ? import nixpkgsPinned.nixpkgs-unstable { config = {}; overlays = []; } +}: let self = { spark-wallet = pkgs.callPackage ./spark-wallet { }; liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { }; @@ -16,7 +22,7 @@ let self = { packageOverrides = import ./python-packages self; }).pkgs; - pinned = import ./pinned.nix; + pinned = import ./pinned.nix pkgs pkgsUnstable; modulesPkgs = self // self.pinned; }; in self diff --git a/pkgs/pinned.nix b/pkgs/pinned.nix index 0851d03..3c4337b 100644 --- a/pkgs/pinned.nix +++ b/pkgs/pinned.nix @@ -1,12 +1,6 @@ -let - nixpkgsPinned = import ./nixpkgs-pinned.nix; - nixpkgsStable = import nixpkgsPinned.nixpkgs { config = {}; overlays = []; }; - nixpkgsUnstable = import nixpkgsPinned.nixpkgs-unstable { config = {}; overlays = []; }; - nixBitcoinPkgsStable = import ./. { pkgs = nixpkgsStable; }; - nixBitcoinPkgsUnstable = import ./. { pkgs = nixpkgsUnstable; }; -in +pkgs: pkgsUnstable: { - inherit (nixpkgsUnstable) + inherit (pkgsUnstable) bitcoin bitcoind charge-lnd @@ -21,8 +15,5 @@ in lightning-loop lightning-pool; - inherit nixpkgsStable nixpkgsUnstable; - - stable = nixBitcoinPkgsStable; - unstable = nixBitcoinPkgsUnstable; + inherit pkgs pkgsUnstable; } diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index d8cbc5b..66e0961 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -4,8 +4,6 @@ let joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; }; clightningPkg = pkg: callPackage pkg { inherit (nbPkgs.pinned) clightning; }; - - unstable = (import ../nixpkgs-pinned.nix).nixpkgs-unstable; in { bencoderpyx = callPackage ./bencoderpyx {}; coincurve = callPackage ./coincurve {}; From f7c2133250cef7d63857ab48bee63d62dacbb18e Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 16 Aug 2021 10:42:07 +0200 Subject: [PATCH 2/4] add flake support This change is fully backwards compatible. We continue to use the standard non-flake evaluation mode in our examples and internal tooling until the flakes design has stabilized. 'clightning-plugins = pkgs.recurseIntoAttrs' in pkgs/default.nix is needed by flake-utils.lib.flattenTree in flake.nix. It transforms the packages in `clightning-plugins` to top-level packages named like `clightning-plugins/summary`. (The flake attr `packages` must be a non-nested attrset of derivations.) --- examples/README.md | 10 +++- examples/flakes/flake.nix | 49 ++++++++++++++++++ flake.lock | 60 +++++++++++++++++++++ flake.nix | 106 ++++++++++++++++++++++++++++++++++++++ pkgs/default.nix | 2 +- pkgs/nixpkgs-pinned.nix | 23 +++++---- test/run-tests.sh | 9 ++++ 7 files changed, 246 insertions(+), 13 deletions(-) create mode 100644 examples/flakes/flake.nix create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/examples/README.md b/examples/README.md index 445f846..20e3d0f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -69,9 +69,17 @@ c systemctl status bitcoind ``` See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation. - ### Real-world example Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org to see the configuration of a nix-bitcoin node that's used in production. The commands in `shell.nix` allow you to locally run the node in a VM or container. + +### Flakes + +Flakes make it easy to include `nix-bitcoin` in an existing NixOS config. +The [flakes example](./flakes/flake.nix) shows how to use `nix-bitcoin` as an input to a system flake. + +Run `nix run` or `nix run .#vm` from the nix-bitcoin root directory to start an example +nix-bitcoin node VM. +This command is defined by the nix-bitcoin flake (in [flake.nix](../flake.nix)). diff --git a/examples/flakes/flake.nix b/examples/flakes/flake.nix new file mode 100644 index 0000000..aaaff29 --- /dev/null +++ b/examples/flakes/flake.nix @@ -0,0 +1,49 @@ +{ + description = "A basic nix-bitcoin node"; + + inputs.nix-bitcoin.url = "github:fort-nix/nix-bitcoin"; + + outputs = { self, nix-bitcoin }: { + + nixosConfigurations.mynode = nix-bitcoin.inputs.nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ## Note: + ## If you use a custom nixpkgs version for evaluating your system, + ## consider using `withLockedPkgs` instead of `withSystemPkgs` to use the exact + ## pkgs versions for nix-bitcoin services that are tested by nix-bitcoin. + ## The downsides are increased evaluation times and increased system + ## closure size. + # + # nix-bitcoin.nixosModules.withLockedPkgs + nix-bitcoin.nixosModules.withSystemPkgs + + ## Optional: + ## Import the secure-node preset, an opinionated config to enhance security + ## and privacy. + # + # "${nix-bitcoin}/modules/presets/secure-node.nix" + + { + nix-bitcoin.generateSecrets = true; + + services.bitcoind.enable = true; + + # When using nix-bitcoin as part of a larger NixOS configuration, set the following to enable + # interactive access to nix-bitcoin features (like bitcoin-cli) for your system's main user + nix-bitcoin.operator = { + enable = true; + name = "main"; # Set this to your system's main user + }; + + # The system's main unprivileged user. This setting is usually part of your + # existing NixOS configuration. + users.users.main = { + isNormalUser = true; + password = "a"; + }; + } + ]; + }; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..00d2d9f --- /dev/null +++ b/flake.lock @@ -0,0 +1,60 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1623875721, + "narHash": "sha256-A8BU7bjS5GirpAUv4QA+QnJ4CceLHkcXdRp4xITDB0s=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "f7e004a55b120c02ecb6219596820fcd32ca8772", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1628865210, + "narHash": "sha256-dB3IA8AYUQDXH+Xy/6nbv4QpIbVl88DphbcxJSUYiX4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a445f5829889959d65ad65e5c961d5c67e1cd677", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-21.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgsUnstable": { + "locked": { + "lastModified": 1628779307, + "narHash": "sha256-ErivbgE5SGvNdFpq+Q50lw1JGGmvxBR/d71aW41S+1A=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4138cbd913fad85073e59007710e3f083d0eb7c6", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgsUnstable": "nixpkgsUnstable" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..12828a1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,106 @@ +{ + description = '' + A collection of Nix packages and NixOS modules for easily + installing full-featured Bitcoin nodes with an emphasis on security. + ''; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05"; + nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, nixpkgsUnstable, flake-utils }: + let + supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ]; + in + rec { + mkNbPkgs = { + system + , pkgs ? import nixpkgs { inherit system; } + , pkgsUnstable ? import nixpkgsUnstable { inherit system; } + }: + import ./pkgs { inherit pkgs pkgsUnstable; }; + + overlay = final: prev: let + nbPkgs = mkNbPkgs { inherit (final) system; pkgs = final; }; + in removeAttrs nbPkgs [ "pinned" "nixops19_09" "krops" ]; + + nixosModules = { + # Uses the default system pkgs for nix-bitcoin.pkgs + withSystemPkgs = { pkgs, ... }: { + imports = [ ./modules/modules.nix ]; + nix-bitcoin.pkgs = (mkNbPkgs { inherit (pkgs) system; inherit pkgs; }).modulesPkgs; + }; + + # Uses the nixpkgs version locked by this flake for nix-bitcoin.pkgs. + # More stable, but slightly slower to evaluate and needs more space if the + # locked and the system nixpkgs versions differ. + withLockedPkgs = { config, ... }: { + imports = [ ./modules/modules.nix ]; + nix-bitcoin.pkgs = (mkNbPkgs { inherit (config.nixpkgs) system; }).modulesPkgs; + }; + }; + + defaultTemplate = { + description = "Basic node template"; + path = ./examples/flakes; + }; + + } // (flake-utils.lib.eachSystem supportedSystems (system: + let + pkgs = import nixpkgs { inherit system; }; + + mkVMScript = vm: pkgs.writers.writeBash "run-vm" '' + set -euo pipefail + export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-vm.XXX) + trap "rm -rf $TMPDIR" EXIT + export NIX_DISK_IMAGE=$TMPDIR/nixos.qcow2 + QEMU_OPTS="-smp $(nproc) -m 1500" ${vm}/bin/run-*-vm + ''; + in rec { + nbPkgs = self.mkNbPkgs { inherit system pkgs; }; + + packages = flake-utils.lib.flattenTree (removeAttrs nbPkgs [ + "pinned" "modulesPkgs" "nixops19_09" "krops" + ]) // { + runVM = mkVMScript packages.vm; + + # This is a simple demo VM. + # See ./examples/flakes/flake.nix on how to use nix-bitcoin with flakes. + vm = let + nix-bitcoin = self; + in + (import "${nixpkgs}/nixos" { + inherit system; + configuration = { + imports = [ + nix-bitcoin.nixosModules.withSystemPkgs + "${nix-bitcoin}/modules/presets/secure-node.nix" + ]; + + nix-bitcoin.generateSecrets = true; + services.clightning.enable = true; + # For faster startup in offline VMs + services.clightning.extraConfig = "disable-dns"; + + nixpkgs.pkgs = pkgs; + virtualisation.graphics = false; + services.getty.autologinUser = "root"; + nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; + }; + }).vm; + }; + + defaultApp = apps.vm; + + apps = { + # Run a basic nix-bitcoin node in a VM + vm = { + type = "app"; + program = toString packages.runVM; + }; + }; + } + )); +} diff --git a/pkgs/default.nix b/pkgs/default.nix index 039cd08..1e41b01 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -14,7 +14,7 @@ let self = { krops = import ./krops { }; netns-exec = pkgs.callPackage ./netns-exec { }; extra-container = pkgs.callPackage ./extra-container { }; - clightning-plugins = import ./clightning-plugins pkgs self.nbPython3Packages; + clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages); clboss = pkgs.callPackage ./clboss { }; secp256k1 = pkgs.callPackage ./secp256k1 { }; diff --git a/pkgs/nixpkgs-pinned.nix b/pkgs/nixpkgs-pinned.nix index 0b57e7e..d3f3acc 100644 --- a/pkgs/nixpkgs-pinned.nix +++ b/pkgs/nixpkgs-pinned.nix @@ -1,19 +1,20 @@ let - fetch = { rev, sha256 }: + fetchNixpkgs = { rev, sha256 }: builtins.fetchTarball { url = "https://github.com/nixos/nixpkgs/archive/${rev}.tar.gz"; inherit sha256; }; + + fetch = input: let + inherit (input) locked; + in fetchNixpkgs { + inherit (locked) rev; + sha256 = locked.narHash; + }; + + lockedInputs = (builtins.fromJSON (builtins.readFile ../flake.lock)).nodes; in { - # To update, run ../helper/fetch-channel REV - nixpkgs = fetch { - # nixos-21.05 (2021-08-14) - rev = "a445f5829889959d65ad65e5c961d5c67e1cd677"; - sha256 = "0zl930jjacdphplw1wv5nlhjk15zvflzzwp53zbh0l8qq01wh7bl"; - }; - nixpkgs-unstable = fetch { - rev = "4138cbd913fad85073e59007710e3f083d0eb7c6"; - sha256 = "0l7vaa6mnnmxfxzi9i5gd4c4j3cpfh7gjsjsfk6nnj1r05pazf0j"; - }; + nixpkgs = fetch lockedInputs.nixpkgs; + nixpkgs-unstable = fetch lockedInputs.nixpkgsUnstable; } diff --git a/test/run-tests.sh b/test/run-tests.sh index 7990308..241f142 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -281,9 +281,18 @@ examples() { (cd "$scriptDir/../examples" && nix-shell --run "$script") } +flake() { + if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then + echo "Skipping flake test. Nix flake support is not enabled." + else + nix flake check "$scriptDir/.." + fi +} + all() { buildable examples + flake } # An alias for buildTest From 87df809a8808c1aaddeb82397ac3f37816cdee1b Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 16 Aug 2021 10:42:08 +0200 Subject: [PATCH 3/4] add helper/update-flake.sh This greatly simplifies updating nixpkgs. See the comment at the top of update-flake.sh for a description. --- helper/update-flake.nix | 77 +++++++++++++++++++++++++++++++++++++++++ helper/update-flake.sh | 45 ++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 helper/update-flake.nix create mode 100755 helper/update-flake.sh diff --git a/helper/update-flake.nix b/helper/update-flake.nix new file mode 100644 index 0000000..23e2012 --- /dev/null +++ b/helper/update-flake.nix @@ -0,0 +1,77 @@ +{ prevVersions ? null }: +let + flake = builtins.getFlake (toString ../.); + inherit (flake.inputs.nixpkgs) lib; +in rec { + # Convert the list of pinned pkgs to an attrset with form + # { + # stable = { bitcoind = "0.21.1"; ... }; + # unstable = { btcpayserver = "1.2.1"; ... }; + # } + # A pinned pkg is added to `stable` if the stable and unstable pkg versions + # are identical. + versions = let + pinned = flake.nbPkgs.x86_64-linux.pinned; + pinnedPkgs = lib.filterAttrs (n: v: lib.isDerivation v) pinned; + stable = pinned.pkgs; + unstable = pinned.pkgsUnstable; + isStable = builtins.partition (pkgName: + !(unstable ? "${pkgName}") || + ((stable ? "${pkgName}") && stable.${pkgName}.version == unstable.${pkgName}.version) + ) (builtins.attrNames pinnedPkgs); + in { + stable = lib.genAttrs isStable.right (pkgName: stable.${pkgName}.version); + unstable = lib.genAttrs isStable.wrong (pkgName: unstable.${pkgName}.version); + }; + + showUpdates = let + prev = builtins.fromJSON prevVersions; + prevPkgs = prev.stable // prev.unstable; + mapFilter = f: xs: lib.remove null (map f xs); + + mkChanges = title: pkgs: + let + lines = mapFilter (pkgName: + let + version = pkgs.${pkgName}; + prevVersion = prevPkgs.${pkgName}; + in + if version != prevVersion then + "${pkgName}: ${prevVersion} -> ${version}" + else + null + ) (builtins.attrNames pkgs); + in + if lines == [] then + null + else + builtins.concatStringsSep "\n" ([title] ++ lines); + + changes = [ + (mkChanges "Pkg updates in nixpkgs stable:" versions.stable) + (mkChanges "Pkg updates in nixpkgs unstable:" versions.unstable) + ]; + + allChanges = builtins.concatStringsSep "\n\n" (lib.remove null changes); + in + if allChanges == "" then + "No pkg version updates." + else + allChanges; + + pinnedFile = let + toLines = pkgs: builtins.concatStringsSep "\n " (builtins.attrNames pkgs); + in '' + # This file is generated by ../helper/update-flake.nix + pkgs: pkgsUnstable: + { + inherit (pkgs) + ${toLines versions.stable}; + + inherit (pkgsUnstable) + ${toLines versions.unstable}; + + inherit pkgs pkgsUnstable; + } + ''; +} diff --git a/helper/update-flake.sh b/helper/update-flake.sh new file mode 100755 index 0000000..939786f --- /dev/null +++ b/helper/update-flake.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script does the following: +# - Update all flake inputs, including nixpkgs +# - Print version updates of pinned pkgs like so: +# Pkg updates in nixpkgs unstable: +# bitcoin: 0.20.0 -> 0.21.1 +# btcpayserver: 1.1.0 -> 1.1.2 +# - Write ../pkgs/pinned.nix: +# Packages for which the stable und unstable versions are identical are +# pinned to stable. +# All other pkgs are pinned to unstable. + +# cd to script dir +cd "${BASH_SOURCE[0]%/*}" + +if [[ $(nix flake 2>&1) != *"requires a sub-command"* ]]; then + echo "Error. This script requires nix flake support." + echo "https://nixos.wiki/wiki/Flakes#Installing_flakes" + exit 1 +fi + +if [[ ${1:-} != -f ]] && ! git diff --quiet ../flake.{nix,lock}; then + echo "error: flake.nix/flake.lock have changes. Run with option -f to ignore." + exit 1 +fi + +versions=$(nix eval --json -f update-flake.nix versions) + +## Uncomment the following to generate a version change message for testing +# versions=$(echo "$versions" | sed 's|1|0|g') + +nix flake update .. + +echo +nix eval --raw -f update-flake.nix --argstr prevVersions "$versions" showUpdates; echo + +pinned=../pkgs/pinned.nix +pinnedSrc=$(nix eval --raw -f update-flake.nix --argstr prevVersions "$versions" pinnedFile) +if [[ $pinnedSrc != $(cat "$pinned") ]]; then + echo "$pinnedSrc" > "$pinned" + echo + echo "Updated pinned.nix" +fi From dde04f8cbeb93b2bbea5849d74c5856f36a7289a Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 16 Aug 2021 10:42:09 +0200 Subject: [PATCH 4/4] update nixpkgs-unstable Includes: btcpayserver: 1.1.2 -> 1.2.0 lightning-loop: 0.14.2-beta -> 0.15.0-beta nbxplorer: 2.1.52 -> 2.1.58 --- flake.lock | 6 +++--- pkgs/pinned.nix | 15 +++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 00d2d9f..fe469b2 100644 --- a/flake.lock +++ b/flake.lock @@ -33,11 +33,11 @@ }, "nixpkgsUnstable": { "locked": { - "lastModified": 1628779307, - "narHash": "sha256-ErivbgE5SGvNdFpq+Q50lw1JGGmvxBR/d71aW41S+1A=", + "lastModified": 1628934079, + "narHash": "sha256-CEYsKXNYprs/TvmB7ppkYMALXnfhEw6lg5VaEXgpoec=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4138cbd913fad85073e59007710e3f083d0eb7c6", + "rev": "fd9984fd9a950686e7271ecf01893987a42cdf14", "type": "github" }, "original": { diff --git a/pkgs/pinned.nix b/pkgs/pinned.nix index 3c4337b..4059597 100644 --- a/pkgs/pinned.nix +++ b/pkgs/pinned.nix @@ -1,19 +1,22 @@ +# This file is generated by ../helper/update-flake.nix pkgs: pkgsUnstable: { - inherit (pkgsUnstable) + inherit (pkgs) bitcoin bitcoind + lndconnect; + + inherit (pkgsUnstable) + btcpayserver charge-lnd clightning - lnd - lndconnect - nbxplorer - btcpayserver electrs elementsd hwi lightning-loop - lightning-pool; + lightning-pool + lnd + nbxplorer; inherit pkgs pkgsUnstable; }