From af115d746b2f96793cf39a1d5c1b11393da9b493 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 14:57:39 +0200 Subject: [PATCH 1/9] internal scripts: use pinned, cached pkgs Instead of setting up the script PATH via nix-shell, use `nix shell` with inputs from the nix-bitcoin flake. Advantages: - Uses the nixpkgs version from the nix-bitcoin flake instead of `` from the user env (NIX_PATH), so the script runtime env is reproducible. - The pkg derivations for the runtime env are cached, which greatly increases script startup speed. This commit was generated by running the following script inside the repo root dir: def transform(path, src) if src =~ /#! *nix-shell +-i +bash +-p +(.*)/ pkgs = $1 if src =~ /^.*?(set -e.*?pipefail)\n/ set_statement = $1 src.sub!($&, '') end src.sub!(/\A.*?#! *nix-shell.*?\n/m, '') parents = ([ '..' ] * (path.split('/').count - 1)).join('/') [ '#!/usr/bin/env bash', *set_statement, %(. "${BASH_SOURCE[0]%/*}/#{parents}/helper/run-in-nix-env" "#{pkgs}" "$@"), nil, src ].join("\n") end end Dir['**/*.sh'].each do |f| src = File.read(f) if new_src = transform(f, src) puts "Changed file #{f}" File.write(f, new_src) end end --- .cirrus.yml | 2 +- helper/run-in-nix-env | 20 +++++++++++++++++++ pkgs/clightning-plugins/get-sha256.sh | 4 ++-- pkgs/clightning-rest/generate.sh | 4 ++-- pkgs/joinmarket/get-sha256.sh | 4 ++-- pkgs/krops/fetch-release.sh | 4 ++-- pkgs/lndinit/get-sha256.sh | 4 ++-- .../python-bitcointx/get-sha256.sh | 4 ++-- pkgs/rtl/generate.sh | 4 ++-- pkgs/spark-wallet/generate.sh | 4 ++-- test/run-tests.sh | 2 +- test/shellcheck.sh | 1 + 12 files changed, 39 insertions(+), 18 deletions(-) create mode 100644 helper/run-in-nix-env diff --git a/.cirrus.yml b/.cirrus.yml index 3b56692..430078f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -39,4 +39,4 @@ task: - name: shellcheck build_script: - - nix shell --inputs-from . nixpkgs#{shellcheck,findutils,gnugrep} -c ./test/shellcheck.sh + - ./test/shellcheck.sh diff --git a/helper/run-in-nix-env b/helper/run-in-nix-env new file mode 100644 index 0000000..d2e0976 --- /dev/null +++ b/helper/run-in-nix-env @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Usage: +# source "path/to/run-in-nix-env" "pkg1 pkg2 ..." "$@" +# +# Runs the calling script with pkgs1, pkg2, ... in PATH. +# The packages are sourced from the nixpkgs input of the nix-bitcoin root flake. + +if [[ ${_NIX_BITCOIN_ENV_PKGS:-} != "$1" ]]; then + export _NIX_BITCOIN_ENV_PKGS=$1 + shift + + _shell_pkgs=() + for _pkg in $_NIX_BITCOIN_ENV_PKGS; do + _shell_pkgs+=("nixpkgs#$_pkg") + done + # BASH_SOURCE[0] is this file + # BASH_SOURCE[-1] is the root src file + exec nix shell --inputs-from "${BASH_SOURCE[0]%/*}/.." "${_shell_pkgs[@]}" -c "${BASH_SOURCE[-1]}" "$@" +fi diff --git a/pkgs/clightning-plugins/get-sha256.sh b/pkgs/clightning-plugins/get-sha256.sh index 886d5f8..b66906f 100755 --- a/pkgs/clightning-plugins/get-sha256.sh +++ b/pkgs/clightning-plugins/get-sha256.sh @@ -1,6 +1,6 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p git +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "git" "$@" archive_hash () { repo=$1 diff --git a/pkgs/clightning-rest/generate.sh b/pkgs/clightning-rest/generate.sh index 038be02..c03bdf3 100755 --- a/pkgs/clightning-rest/generate.sh +++ b/pkgs/clightning-rest/generate.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p gnupg wget gnused +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "gnupg wget gnused" "$@" version="0.9.0" repo=https://github.com/Ride-The-Lightning/c-lightning-REST diff --git a/pkgs/joinmarket/get-sha256.sh b/pkgs/joinmarket/get-sha256.sh index 111aa68..1a36b12 100755 --- a/pkgs/joinmarket/get-sha256.sh +++ b/pkgs/joinmarket/get-sha256.sh @@ -1,6 +1,6 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p git gnupg +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "git gnupg" "$@" TMPDIR="$(mktemp -d -p /tmp)" trap 'rm -rf $TMPDIR' EXIT diff --git a/pkgs/krops/fetch-release.sh b/pkgs/krops/fetch-release.sh index 8fa5c5a..44a56b9 100755 --- a/pkgs/krops/fetch-release.sh +++ b/pkgs/krops/fetch-release.sh @@ -1,6 +1,6 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p git +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "git" "$@" archive_hash () { repo=$1 diff --git a/pkgs/lndinit/get-sha256.sh b/pkgs/lndinit/get-sha256.sh index cca5bca..7b3d310 100755 --- a/pkgs/lndinit/get-sha256.sh +++ b/pkgs/lndinit/get-sha256.sh @@ -1,6 +1,6 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p git gnupg curl jq +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "git gnupg curl jq" "$@" TMPDIR=$(mktemp -d -p /tmp) trap 'rm -rf $TMPDIR' EXIT diff --git a/pkgs/python-packages/python-bitcointx/get-sha256.sh b/pkgs/python-packages/python-bitcointx/get-sha256.sh index 745ffb1..ce70cc8 100755 --- a/pkgs/python-packages/python-bitcointx/get-sha256.sh +++ b/pkgs/python-packages/python-bitcointx/get-sha256.sh @@ -1,6 +1,6 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p git gnupg +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../../helper/run-in-nix-env" "git gnupg" "$@" TMPDIR=$(mktemp -d -p /tmp) trap 'rm -rf $TMPDIR' EXIT diff --git a/pkgs/rtl/generate.sh b/pkgs/rtl/generate.sh index 0e19d44..937c0d2 100755 --- a/pkgs/rtl/generate.sh +++ b/pkgs/rtl/generate.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p gnupg wget gnused +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "gnupg wget gnused" "$@" version="0.13.1" repo=https://github.com/Ride-The-Lightning/RTL diff --git a/pkgs/spark-wallet/generate.sh b/pkgs/spark-wallet/generate.sh index 50c3a83..ecbbd65 100755 --- a/pkgs/spark-wallet/generate.sh +++ b/pkgs/spark-wallet/generate.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p nodePackages.node2nix gnupg wget jq moreutils gnused +#!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../../helper/run-in-nix-env" "nodePackages.node2nix gnupg wget jq moreutils gnused" "$@" TMPDIR=$(mktemp -d -p /tmp) trap 'rm -rf $TMPDIR' EXIT diff --git a/test/run-tests.sh b/test/run-tests.sh index 54f25a9..b380853 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -322,7 +322,7 @@ examples() { shellcheck() { if ! checkFlakeSupport "shellcheck"; then return; fi - nix shell --inputs-from "$scriptDir/.." nixpkgs#shellcheck -c "$scriptDir/shellcheck.sh" + "$scriptDir/shellcheck.sh" } all() { diff --git a/test/shellcheck.sh b/test/shellcheck.sh index 822f5d9..f6b697e 100755 --- a/test/shellcheck.sh +++ b/test/shellcheck.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +. "${BASH_SOURCE[0]%/*}/../helper/run-in-nix-env" "shellcheck findutils gnugrep" "$@" cd "${BASH_SOURCE[0]%/*}/.." { From 3a65dc0dc954273cd6a212e880449fade3e58eaf Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 14:57:40 +0200 Subject: [PATCH 2/9] docs/configuration: improve section `Remote bitcoind` --- docs/configuration.md | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b0f151c..878bf45 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -214,18 +214,15 @@ services.bitcoind = { }; ``` -If a `secure-node.nix` or `tor-enable.nix` preset is imported in your -configuration or a `tor.enforce` option is explicitly enabled, you also need to -allow remote connections for **every** service which needs to connect to the -remote bitcoind: - -``` -systemd.services..serviceConfig = { - IPAddressAllow = [ ${services.bitcoind.rpc.address} ]; -}; +For each service that connects to bitcoind and has option +`services..tor.enforce` enabled (either explicitly or by importing +`secure-node.nix` or `enable-tor.nix`), you need to +allow the remote bitcoind connection: +```nix +systemd.services..serviceConfig.IPAddressAllow = [ ${services.bitcoind.rpc.address} ]; ``` -> Please note that configuration above applies only if the remote bitcoind **is +> The above configuration is only required if the remote bitcoind **is > not** accessed via Tor. From f2a4d1fe650637a245b5c5ae35ea8ddf49d8504b Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 14:57:41 +0200 Subject: [PATCH 3/9] docs/security: fix typo --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 8ec3485..e6faf95 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -35,7 +35,7 @@ amounts. The nix-bitcoin developers [listed above](#reporting-a-vulnerability) each hold one key to the multisig address and collectively form the nix-bitcoin developer -quorum: +quorum. ### Eligible Vulnerabilities From cd3878e35755a80a1f5e1d94e91b469807be3d23 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 14:57:42 +0200 Subject: [PATCH 4/9] delete modules/default.nix This file has been missing some modules for quite some time. Remove it to simplify maintenance. --- default.nix | 4 +--- modules/default.nix | 14 -------------- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 modules/default.nix diff --git a/default.nix b/default.nix index 14030d9..660779e 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,3 @@ { pkgs ? import {} }: -(import ./pkgs { inherit pkgs; }) // { - modules = import ./modules; -} +import ./pkgs { inherit pkgs; } diff --git a/modules/default.nix b/modules/default.nix deleted file mode 100644 index f173f9a..0000000 --- a/modules/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - modules = ./modules.nix; - bitcoind = ./bitcoind.nix; - clightning = ./clightning.nix; - default = ./default.nix; - electrs = ./electrs.nix; - liquid = ./liquid.nix; - presets.secure-node = ./presets/secure-node.nix; - rtl = ./rtl.nix; - spark-wallet = ./spark-wallet.nix; - lnd = ./lnd.nix; - charge-lnd = ./charge-lnd.nix; - joinmarket = ./joinmarket.nix; -} From 1713e016a031c8d366af44c6a47ea8084ca9dda5 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 14:57:43 +0200 Subject: [PATCH 5/9] clightning-replication: clarify comment about FUSE mounts --- modules/clightning-replication.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/clightning-replication.nix b/modules/clightning-replication.nix index 1cf58da..fe482ee 100644 --- a/modules/clightning-replication.nix +++ b/modules/clightning-replication.nix @@ -143,8 +143,11 @@ in { # We can't simply set `destDir` here because it might point to # a FUSE mount. # FUSE mounts can only be set up as `ReadWritePaths` by systemd when they - # are accessible by root. This would require FUSE-mounting with option - # `allow_other`. + # are accessible by root. + # But FUSE mounts are only accessible by the mounting user and + # not by root. + # (This could be circumvented by FUSE-mounting `destDir` with option `allow_other`, + # but this would grant access to all users.) (if useMounts then mountsDir else localDir) ]; }; From 030be842c847aa45679010aa47f519732ad7fa9d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 14:57:44 +0200 Subject: [PATCH 6/9] secrets: use fetchurl derivation This moves fetching from eval time to build time, which is a better fit for fetching build time resources. --- modules/secrets/secrets.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/secrets/secrets.nix b/modules/secrets/secrets.nix index 032e1c1..3a5a10a 100644 --- a/modules/secrets/secrets.nix +++ b/modules/secrets/secrets.nix @@ -73,7 +73,7 @@ let generateSecretsScript = mkOption { internal = true; default = let - rpcauthSrc = builtins.fetchurl { + rpcauthSrc = pkgs.fetchurl { url = "https://raw.githubusercontent.com/bitcoin/bitcoin/d6cde007db9d3e6ee93bd98a9bbfdce9bfa9b15b/share/rpcauth/rpcauth.py"; sha256 = "189mpplam6yzizssrgiyv70c9899ggh8cac76j4n7v0xqzfip07n"; }; From 4e70db14bcebf24449146d97e7c77b789be0de95 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 22 Aug 2022 22:44:21 +0200 Subject: [PATCH 7/9] README: fix capitalization --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b51aa0..b4439d8 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ top left corner of the documents. -* [NixOS Options Search](https://search.nixos.org/flakes?channel=unstable&sort=relevance&type=options&query=bitcoin) +* [NixOS options search](https://search.nixos.org/flakes?channel=unstable&sort=relevance&type=options&query=bitcoin) * [Hardware requirements](docs/hardware.md) * [Installation](docs/install.md) * [Configuration and maintenance](docs/configuration.md) From d9eecb7d4c23a38e9eb3a31a434e7139fed57f86 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 27 Aug 2022 13:39:33 +0200 Subject: [PATCH 8/9] nixos-search/flake: remove unused output --- test/nixos-search/flake.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/test/nixos-search/flake.nix b/test/nixos-search/flake.nix index 76eb49e..113a71a 100644 --- a/test/nixos-search/flake.nix +++ b/test/nixos-search/flake.nix @@ -8,6 +8,5 @@ # Used by ./ci-test.sh inherit (nixos-search.inputs.nixpkgs) legacyPackages; - nixpkgsPath = toString nixos-search.inputs.nixpkgs; }; } From 67a573dae86ae43d30395864351b81ec03c8bef8 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 28 Aug 2022 17:52:47 +0200 Subject: [PATCH 9/9] create-git-repo.sh: fix amend error if there's no branch checked out In this case, `git commit --amend` fails. The state of a .git repo existing without a branch being checked out can be reached through the following sequence of events (which I just seem to have triggered): 1. The script calls `git init` 2. The user interrupts the script before `git commit` is finished --- test/lib/create-git-repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/create-git-repo.sh b/test/lib/create-git-repo.sh index 110dfdf..3549904 100644 --- a/test/lib/create-git-repo.sh +++ b/test/lib/create-git-repo.sh @@ -4,7 +4,7 @@ cd "$scriptDir/.." amend=--amend - if [[ ! -e .git ]]; then + if [[ ! -e .git ]] || ! git rev-parse HEAD 2>/dev/null; then git init amend= fi