From 29d2ffc111ccf553314101dec44a1d9e6fd650dd Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 30 Apr 2022 15:35:45 +0200 Subject: [PATCH 1/9] defaultHardening: allow `clone3` system call clone3 is the latest version of the clone system call, which is already allowed. clone3 is required by nbxplorer 2.3.20. --- pkgs/lib.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/lib.nix b/pkgs/lib.nix index 5ac6e6e..6919ae6 100644 --- a/pkgs/lib.nix +++ b/pkgs/lib.nix @@ -33,7 +33,7 @@ let self = { # @system-service whitelist and docker seccomp blacklist (except for "clone" # which is a core requirement for systemd services) # @system-service is defined in src/shared/seccomp-util.c (systemd source) - SystemCallFilter = [ "@system-service" "~add_key clone3 kcmp keyctl mbind move_pages name_to_handle_at personality process_vm_readv process_vm_writev request_key set_mempolicy setns unshare userfaultfd" ]; + SystemCallFilter = [ "@system-service" "~add_key kcmp keyctl mbind move_pages name_to_handle_at personality process_vm_readv process_vm_writev request_key set_mempolicy setns unshare userfaultfd" ]; SystemCallArchitectures = "native"; }; From 7de56b019fe34872dbfb77a46ba5fadd15e1dc2d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 30 Apr 2022 15:35:46 +0200 Subject: [PATCH 2/9] nbxplorer: use postgresql The former DBTrie backend has been deprecated. --- examples/configuration.nix | 2 +- modules/backups.nix | 4 +++- modules/btcpayserver.nix | 20 ++++++++++++++------ modules/versioning.nix | 11 +++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/examples/configuration.nix b/examples/configuration.nix index 84d0c0e..93b9f14 100644 --- a/examples/configuration.nix +++ b/examples/configuration.nix @@ -268,5 +268,5 @@ # The nix-bitcoin release version that your config is compatible with. # When upgrading to a backwards-incompatible release, nix-bitcoin will display an # an error and provide hints for migrating your config to the new release. - nix-bitcoin.configVersion = "0.0.65"; + nix-bitcoin.configVersion = "0.0.70"; } diff --git a/modules/backups.nix b/modules/backups.nix index 9f2220a..21e0b20 100644 --- a/modules/backups.nix +++ b/modules/backups.nix @@ -121,6 +121,8 @@ in { fi ''; - services.backups.postgresqlDatabases = mkIf config.services.btcpayserver.enable [ "btcpaydb" ]; + services.backups.postgresqlDatabases = mkIf config.services.btcpayserver.enable ( + [ "btcpaydb" ] ++ optional cfg.with-bulk-data "nbxplorer" + ); }; } diff --git a/modules/btcpayserver.nix b/modules/btcpayserver.nix index 2b55e5b..8f5f92a 100644 --- a/modules/btcpayserver.nix +++ b/modules/btcpayserver.nix @@ -136,11 +136,17 @@ in { }; services.postgresql = { enable = true; - ensureDatabases = [ "btcpaydb" ]; - ensureUsers = [{ - name = cfg.btcpayserver.user; - ensurePermissions."DATABASE btcpaydb" = "ALL PRIVILEGES"; - }]; + ensureDatabases = [ "btcpaydb" "nbxplorer" ]; + ensureUsers = [ + { + name = cfg.btcpayserver.user; + ensurePermissions."DATABASE btcpaydb" = "ALL PRIVILEGES"; + } + { + name = cfg.nbxplorer.user; + ensurePermissions."DATABASE nbxplorer" = "ALL PRIVILEGES"; + } + ]; }; systemd.tmpfiles.rules = [ @@ -162,10 +168,12 @@ in { lbtcrpcurl=http://${nbLib.addressWithPort liquidd.rpc.address liquidd.rpc.port} lbtcnodeendpoint=${nbLib.addressWithPort liquidd.address liquidd.whitelistedPort} ''} + postgres=User ID=${cfg.nbxplorer.user};Host=/run/postgresql;Database=nbxplorer + automigrate=1 ''; in rec { wantedBy = [ "multi-user.target" ]; - requires = [ "bitcoind.service" ] ++ optional cfg.btcpayserver.lbtc "liquidd.service"; + requires = [ "bitcoind.service" "postgresql.service" ] ++ optional cfg.btcpayserver.lbtc "liquidd.service"; after = requires; preStart = '' install -m 600 ${configFile} '${cfg.nbxplorer.dataDir}/settings.config' diff --git a/modules/versioning.nix b/modules/versioning.nix index e1d6761..aa9693f 100644 --- a/modules/versioning.nix +++ b/modules/versioning.nix @@ -202,6 +202,17 @@ let config to your node. ''; } + { + version = "0.0.70"; + condition = config.services.nbxplorer.enable; + message = '' + The nbxplorer database backend has changed from DBTrie to Postgresql. + The new `services.postgresql` database name is `nbxplorer`. + The migration happens automatically after deploying. + Migration time for a large server with a 5GB DBTrie database takes about 40 minutes. + See also: https://github.com/dgarage/NBXplorer/blob/master/docs/Postgres-Migration.md + ''; + } ]; mkOnionServiceChange = service: { From aa21fb454e44afa414a2aefebdce5b38146314e8 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 30 Apr 2022 15:35:47 +0200 Subject: [PATCH 3/9] tests: disable `btcpayserver.lbtc` for regtest Previously, in case of btcpayserver.lbtc on regtest, nbxplorer just printed a log error while trying to generate liquidd regtest blocks. Now nbxplorer exits with an error, so disable lbtc. --- test/tests.nix | 16 ++++++++++++++++ test/tests.py | 8 +------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/test/tests.nix b/test/tests.nix index 193c9f4..190616f 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -95,6 +95,7 @@ let }; # Needed to test macaroon creation environment.systemPackages = mkIfTest "btcpayserver" (with pkgs; [ openssl xxd ]); + test.data.btcpayserver-lbtc = config.services.btcpayserver.lbtc; tests.joinmarket = cfg.joinmarket.enable; tests.joinmarket-yieldgenerator = cfg.joinmarket.yieldgenerator.enable; @@ -264,6 +265,21 @@ let # `validatepegin` is incompatible with regtest services.liquidd.validatepegin = mkForce false; + + # TODO-EXTERNAL: + # Reenable `btcpayserver.lbtc` in regtest (and add test in tests.py) + # when nbxplorer can parse liquidd regtest blocks. + # + # When `btcpayserver.lbtc` is enabled in regtest, nxbplorer tries to + # generate regtest blocks, which fails because no liquidd wallet exists. + # When blocks are pre-generated via `liquidd.postStart`, nbxplorer + # fails to parse the blocks: + # info: NBXplorer.Indexer.LBTC: Full node version detected: 210002 + # info: NBXplorer.Indexer.LBTC: NBXplorer is correctly whitelisted by the node + # fail: NBXplorer.Indexer.LBTC: Unhandled exception in the indexer, retrying in 10 seconds + # System.IO.EndOfStreamException: No more byte to read + # at NBitcoin.BitcoinStream.ReadWriteBytes(Span`1 data) + services.btcpayserver.lbtc = mkForce false; }; ## Examples / debug helper diff --git a/test/tests.py b/test/tests.py index c6cab60..9f2f386 100644 --- a/test/tests.py +++ b/test/tests.py @@ -188,7 +188,7 @@ def _(): def _(): assert_running("nbxplorer") machine.wait_until_succeeds(log_has_string("nbxplorer", "BTC: RPC connection successful")) - if "liquidd" in enabled_tests: + if test_data["btcpayserver-lbtc"]: machine.wait_until_succeeds(log_has_string("nbxplorer", "LBTC: RPC connection successful")) wait_for_open_port(ip("nbxplorer"), 24444) @@ -399,12 +399,6 @@ def _(): succeed("runuser -u operator -- pool orders list") if enabled("btcpayserver"): machine.wait_until_succeeds(log_has_string("nbxplorer", f"BTC: Starting scan at block {num_blocks}")) - # TODO-EXTERNAL: - # nbxplorer 2.2.16 currently fails with with lbtc (liquidd) on regtest - # LBTC: Full node version detected: 210000 - # LBTC: RPC connection successful - # LBTC: Failed to connect to RPC - # System.IO.EndOfStreamException: No more byte to read if "netns-isolation" in enabled_tests: def ip(name): From 408486ac9c159395d603a577ffe251da307eb463 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sat, 30 Apr 2022 15:35:48 +0200 Subject: [PATCH 4/9] tests: update nbxplorer --- test/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests.py b/test/tests.py index 9f2f386..a90bf03 100644 --- a/test/tests.py +++ b/test/tests.py @@ -398,7 +398,7 @@ def _(): ) succeed("runuser -u operator -- pool orders list") if enabled("btcpayserver"): - machine.wait_until_succeeds(log_has_string("nbxplorer", f"BTC: Starting scan at block {num_blocks}")) + machine.wait_until_succeeds(log_has_string("nbxplorer", f"At height: {num_blocks}")) if "netns-isolation" in enabled_tests: def ip(name): From 43ee7668abb79a1b0200b87f29f924add852156e Mon Sep 17 00:00:00 2001 From: Jonas Nick Date: Thu, 5 May 2022 12:50:19 +0000 Subject: [PATCH 5/9] joinmarket: explicitly create no descriptor wallet The default value of `descriptors` has changed from `false` to `true` in Bitcoin Core 23.0. --- modules/joinmarket.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/joinmarket.nix b/modules/joinmarket.nix index c14b40e..6a35aba 100644 --- a/modules/joinmarket.nix +++ b/modules/joinmarket.nix @@ -290,6 +290,7 @@ in { echo "Create watch-only wallet ${cfg.rpcWalletFile}" if ! output=$(${bitcoind.cli}/bin/bitcoin-cli -named createwallet \ wallet_name="${cfg.rpcWalletFile}" \ + descriptors=false \ ${optionalString (!bitcoind.regtest) "disable_private_keys=true"} 2>&1 ); then # Ignore error if bitcoind wallet already exists From ca834cce8416c3a031c27f2bb5064d45cc4ee774 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 5 May 2022 20:43:08 +0200 Subject: [PATCH 6/9] joinmarket: simplify pkgs Remove unused dependencies. --- pkgs/python-packages/jmbitcoin/default.nix | 4 ++-- pkgs/python-packages/jmclient/default.nix | 7 +++---- pkgs/python-packages/jmdaemon/default.nix | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pkgs/python-packages/jmbitcoin/default.nix b/pkgs/python-packages/jmbitcoin/default.nix index c092436..fcc1751 100644 --- a/pkgs/python-packages/jmbitcoin/default.nix +++ b/pkgs/python-packages/jmbitcoin/default.nix @@ -1,4 +1,4 @@ -{ version, src, lib, buildPythonPackage, fetchurl, future, coincurve, urldecode, pyaes, python-bitcointx, joinmarketbase }: +{ version, src, lib, buildPythonPackage, fetchurl, urldecode, pyaes, python-bitcointx, joinmarketbase }: buildPythonPackage rec { pname = "joinmarketbitcoin"; @@ -6,7 +6,7 @@ buildPythonPackage rec { postUnpack = "sourceRoot=$sourceRoot/jmbitcoin"; - propagatedBuildInputs = [ future coincurve urldecode pyaes python-bitcointx ]; + propagatedBuildInputs = [ urldecode pyaes python-bitcointx ]; checkInputs = [ joinmarketbase ]; diff --git a/pkgs/python-packages/jmclient/default.nix b/pkgs/python-packages/jmclient/default.nix index 6b9d8ef..dde369b 100644 --- a/pkgs/python-packages/jmclient/default.nix +++ b/pkgs/python-packages/jmclient/default.nix @@ -1,4 +1,4 @@ -{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon, klein, pyjwt, autobahn, cryptography }: +{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, klein, pyjwt, autobahn }: buildPythonPackage rec { pname = "joinmarketclient"; @@ -6,10 +6,9 @@ buildPythonPackage rec { postUnpack = "sourceRoot=$sourceRoot/jmclient"; - checkInputs = [ joinmarketbitcoin joinmarketdaemon txtorcon ]; + checkInputs = [ joinmarketbitcoin joinmarketdaemon ]; - # configparser may need to be compiled with python_version<"3.2" - propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes klein pyjwt autobahn cryptography ]; + propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes klein pyjwt autobahn ]; patchPhase = '' substituteInPlace setup.py \ diff --git a/pkgs/python-packages/jmdaemon/default.nix b/pkgs/python-packages/jmdaemon/default.nix index 5dd5a7c..c643f94 100644 --- a/pkgs/python-packages/jmdaemon/default.nix +++ b/pkgs/python-packages/jmdaemon/default.nix @@ -1,4 +1,4 @@ -{ version, src, lib, buildPythonPackage, fetchurl, future, txtorcon, cryptography, pyopenssl, libnacl, joinmarketbase }: +{ version, src, lib, buildPythonPackage, fetchurl, txtorcon, cryptography, pyopenssl, libnacl, joinmarketbase }: buildPythonPackage rec { pname = "joinmarketdaemon"; @@ -6,7 +6,7 @@ buildPythonPackage rec { postUnpack = "sourceRoot=$sourceRoot/jmdaemon"; - propagatedBuildInputs = [ future txtorcon cryptography pyopenssl libnacl joinmarketbase ]; + propagatedBuildInputs = [ txtorcon cryptography pyopenssl libnacl joinmarketbase ]; meta = with lib; { description = "Client library for Bitcoin coinjoins"; From 900836fe0d5983192db78c7da5a15a66925cf84d Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 5 May 2022 20:43:09 +0200 Subject: [PATCH 7/9] joinmarket: add private python package set This is a nonfunctional refactoring commit. It's needed because pkg `pyln-proto`, which is introduced in the next commit, requires a different, incompatible version of `cryptography`, which must be placed in a different python package set. --- pkgs/default.nix | 2 +- pkgs/joinmarket/default.nix | 14 ++++++++++++-- pkgs/python-packages/default.nix | 8 +++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkgs/default.nix b/pkgs/default.nix index 672f2b2..e79149e 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -9,7 +9,7 @@ let self = { cl-rest = pkgs.callPackage ./cl-rest { }; clboss = pkgs.callPackage ./clboss { }; clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages); - joinmarket = pkgs.callPackage ./joinmarket { inherit (self) nbPython3Packages; }; + joinmarket = pkgs.callPackage ./joinmarket { nbPythonPackageOverrides = import ./python-packages self; }; lndinit = pkgs.callPackage ./lndinit { }; liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { }; rtl = pkgs.callPackage ./rtl { }; diff --git a/pkgs/joinmarket/default.nix b/pkgs/joinmarket/default.nix index 2ec027e..4ffc890 100644 --- a/pkgs/joinmarket/default.nix +++ b/pkgs/joinmarket/default.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, fetchurl, applyPatches, fetchpatch, python3, nbPython3Packages, pkgs }: +{ stdenv, lib, fetchurl, applyPatches, fetchpatch, python3, nbPythonPackageOverrides, pkgs }: let version = "0.9.5"; @@ -17,7 +17,17 @@ let ]; }; - runtimePackages = with nbPython3Packages; [ + pyPkgs = (python3.override { + packageOverrides = (self: super: let + overrides = nbPythonPackageOverrides self super; + in + overrides // { + cryptography = overrides.cryptography_3_3_2; + } + ); + }).pkgs; + + runtimePackages = with pyPkgs; [ joinmarketbase joinmarketclient joinmarketbitcoin diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index c7162f3..c0a7568 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -30,13 +30,15 @@ in { base58 = callPackage ./specific-versions/base58.nix {}; # cryptography 3.3.2, required by joinmarketdaemon - cryptography = callPackage ./specific-versions/cryptography {}; - cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {}; + # Used in the private python package set for joinmarket (../joinmarket/default.nix) + cryptography_3_3_2 = callPackage ./specific-versions/cryptography { + cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {}; + }; # autobahn 20.12.3, required by joinmarketclient autobahn = callPackage ./specific-versions/autobahn.nix {}; - # tubes 0.2.0, required by klein + # tubes 0.2.0, required by jmclient (via pkg `klein`) tubes = callPackage ./specific-versions/tubes.nix {}; # recommonmark 0.7.1, required by pyln-client From 6bdf0ac3fb485438098b668341dcf205066dcdaf Mon Sep 17 00:00:00 2001 From: Jonas Nick Date: Thu, 5 May 2022 09:55:05 +0000 Subject: [PATCH 8/9] update nixpkgs{,-unstable} This commit does not pass the tests. Fixup is in the next commit. bitcoin: 22.0 -> 23.0 bitcoind: 22.0 -> 23.0 btcpayserver: 1.4.7 -> 1.5.1 clightning: 0.10.2 -> 0.11.0.1 electrs: 0.9.6 -> 0.9.7 hwi: 2.0.2 -> 2.1.0 lightning-loop: 0.17.0-beta -> 0.18.0-beta lnd: 0.14.2-beta -> 0.14.3-beta nbxplorer: 2.2.20 -> 2.3.20 --- flake.lock | 18 +++++++++--------- pkgs/pinned.nix | 8 ++++---- test/nixos-search/flake.lock | 35 ++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/flake.lock b/flake.lock index e9addae..7d91083 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1648297722, - "narHash": "sha256-W+qlPsiZd8F3XkzXOzAoR+mpFqzm3ekQkJNa+PIh1BQ=", + "lastModified": 1649676176, + "narHash": "sha256-OWKJratjt2RW151VUlJPRALb7OU2S5s+f0vLj4o1bHM=", "owner": "numtide", "repo": "flake-utils", - "rev": "0f8662f1319ad6abf89b3380dd2722369fc51ade", + "rev": "a4b154ebbdc88c8498a5c7b01589addc9e9cb678", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1648965846, - "narHash": "sha256-xaO0KS+sgZLYrhaQNjVe6eRcOUIM1mEkAjT+dRbPblU=", + "lastModified": 1651571855, + "narHash": "sha256-KZie6c2J2DUPLjG2PkYSwGLvD6RygA3TxZSPQpctbNI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0aac710801aec4ba545527cf41a5706028fe6271", + "rev": "fd43ce017d4c95f47166d28664a004f57458a0b1", "type": "github" }, "original": { @@ -33,11 +33,11 @@ }, "nixpkgsUnstable": { "locked": { - "lastModified": 1648219316, - "narHash": "sha256-Ctij+dOi0ZZIfX5eMhgwugfvB+WZSrvVNAyAuANOsnQ=", + "lastModified": 1651634615, + "narHash": "sha256-VtvcS61bLh5mIBm9cV3idUHdlfPRFus/NwdJfaj5s8o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "30d3d79b7d3607d56546dd2a6b49e156ba0ec634", + "rev": "abfd31179174133ab8131139d650297bf4da63b7", "type": "github" }, "original": { diff --git a/pkgs/pinned.nix b/pkgs/pinned.nix index 951a70d..3bb2f03 100644 --- a/pkgs/pinned.nix +++ b/pkgs/pinned.nix @@ -2,18 +2,18 @@ pkgs: pkgsUnstable: { inherit (pkgs) - bitcoin - bitcoind charge-lnd - clightning extra-container - hwi lndconnect; inherit (pkgsUnstable) + bitcoin + bitcoind btcpayserver + clightning electrs elementsd + hwi lightning-loop lightning-pool lnd diff --git a/test/nixos-search/flake.lock b/test/nixos-search/flake.lock index 20e2c6e..904dd3b 100644 --- a/test/nixos-search/flake.lock +++ b/test/nixos-search/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1644229661, - "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "lastModified": 1649676176, + "narHash": "sha256-OWKJratjt2RW151VUlJPRALb7OU2S5s+f0vLj4o1bHM=", "owner": "numtide", "repo": "flake-utils", - "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "rev": "a4b154ebbdc88c8498a5c7b01589addc9e9cb678", "type": "github" }, "original": { @@ -15,17 +15,34 @@ "type": "github" } }, + "nixos-org-configurations": { + "flake": false, + "locked": { + "lastModified": 1648674978, + "narHash": "sha256-7lVlPb9/Lu3M9xPzwdvtgqkeQI4FAMv+T3v+yT54ZZ0=", + "owner": "NixOS", + "repo": "nixos-org-configurations", + "rev": "76c0687ac275235142dae7c855aab2885302298c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixos-org-configurations", + "type": "github" + } + }, "nixos-search": { "inputs": { "flake-utils": "flake-utils", + "nixos-org-configurations": "nixos-org-configurations", "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1648850703, - "narHash": "sha256-nZeV3sV3UU9NzJPSGrPN8/clecTqx9byQUtTjS7XMXk=", + "lastModified": 1650836881, + "narHash": "sha256-FKsTNQWq8lv6+Qr6LouEukTsJIGEU51H1cWc1PSOtvU=", "owner": "nixos", "repo": "nixos-search", - "rev": "6ba56bc7e96ce83a5d33868b38acb04ebbc688e1", + "rev": "eea6cd3ffb4d3d39956316d6e976b9d5ce52d508", "type": "github" }, "original": { @@ -36,11 +53,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1642903813, - "narHash": "sha256-0lNfGW8sNfyTrixoQhVG00Drl/ECaf5GbfKAQ1ZDoyE=", + "lastModified": 1650701402, + "narHash": "sha256-XKfstdtqDg+O+gNBx1yGVKWIhLgfEDg/e2lvJSsp9vU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "689b76bcf36055afdeb2e9852f5ecdd2bf483f87", + "rev": "bc41b01dd7a9fdffd32d9b03806798797532a5fe", "type": "github" }, "original": { From f234e59ca5e62a24d8eef0c7490d6f6de3e43c18 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 5 May 2022 20:43:10 +0200 Subject: [PATCH 9/9] nbPython3Packages: fix clightning pkgs Also enable tests for the pyln-* pkgs. --- pkgs/clightning-plugins/default.nix | 5 ++- pkgs/python-packages/coincurve/default.nix | 4 +-- pkgs/python-packages/default.nix | 10 ++++-- pkgs/python-packages/pyln-bolt7/default.nix | 19 ++++------- pkgs/python-packages/pyln-client/default.nix | 19 +++++++++-- pkgs/python-packages/pyln-proto/default.nix | 24 +++++-------- .../specific-versions/base58.nix | 34 ------------------- 7 files changed, 44 insertions(+), 71 deletions(-) delete mode 100644 pkgs/python-packages/specific-versions/base58.nix diff --git a/pkgs/clightning-plugins/default.nix b/pkgs/clightning-plugins/default.nix index eb1d5ac..bf8443b 100644 --- a/pkgs/clightning-plugins/default.nix +++ b/pkgs/clightning-plugins/default.nix @@ -14,7 +14,10 @@ let plugins = with nbPython3Packages; { helpme = { description = "Walks you through setting up a c-lightning node, offering advice for common problems"; }; - monitor = { description = "Helps you analyze the health of your peers and channels"; }; + monitor = { + description = "Helps you analyze the health of your peers and channels"; + extraPkgs = [ packaging ]; + }; prometheus = { description = "Lightning node exporter for the prometheus timeseries server"; extraPkgs = [ prometheus_client ]; diff --git a/pkgs/python-packages/coincurve/default.nix b/pkgs/python-packages/coincurve/default.nix index 042ac67..2e5156d 100644 --- a/pkgs/python-packages/coincurve/default.nix +++ b/pkgs/python-packages/coincurve/default.nix @@ -3,11 +3,11 @@ autoconf, automake, libtool, libffi, requests }: buildPythonPackage rec { pname = "coincurve"; - version = "15.0.0"; + version = "17.0.0"; src = fetchPypi { inherit pname version; - sha256 = "0ras7qb4ib9sik703fcb9f3jrgq7nx5wvdgx9k1pshmrxl8lnlh6"; + hash = "sha256-aNpVr/iYcClS/aPuBP1u1gu2uR+RnGknB4btdmtUi5M"; }; doCheck = false; diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index c0a7568..39fdcc0 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -4,6 +4,8 @@ let joinmarketPkg = pkg: callPackage pkg { inherit (nbPkgs.joinmarket) version src; }; clightningPkg = pkg: callPackage pkg { inherit (nbPkgs.pinned) clightning; }; + + unstable = (import ../nixpkgs-pinned.nix).nixpkgs-unstable; in { bencoderpyx = callPackage ./bencoderpyx {}; coincurve = callPackage ./coincurve {}; @@ -26,15 +28,17 @@ in { ## Specific versions of packages that already exist in nixpkgs - # base58 2.1.0, required by pyln-proto - base58 = callPackage ./specific-versions/base58.nix {}; - # cryptography 3.3.2, required by joinmarketdaemon # Used in the private python package set for joinmarket (../joinmarket/default.nix) cryptography_3_3_2 = callPackage ./specific-versions/cryptography { cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {}; }; + # cryptography 36.0.0, required by pyln-proto. + cryptography = callPackage "${unstable}/pkgs/development/python-modules/cryptography" { + Security = self.darwin.apple_sdk.frameworks.Security; + }; + # autobahn 20.12.3, required by joinmarketclient autobahn = callPackage ./specific-versions/autobahn.nix {}; diff --git a/pkgs/python-packages/pyln-bolt7/default.nix b/pkgs/python-packages/pyln-bolt7/default.nix index 84cb855..16bf920 100644 --- a/pkgs/python-packages/pyln-bolt7/default.nix +++ b/pkgs/python-packages/pyln-bolt7/default.nix @@ -1,22 +1,17 @@ -{ buildPythonPackage, clightning, pyln-proto }: +{ buildPythonPackage, poetry-core, pytestCheckHook, clightning, pyln-proto }: buildPythonPackage rec { pname = "pyln-bolt7"; - - # See fn `bolt_meta` in - # https://github.com/ElementsProject/lightning/blob/master/contrib/pyln-spec/bolt7/setup.py - version = "1.0.2.186"; + # The version is defined here: + # https://github.com/ElementsProject/lightning/blob/master/contrib/pyln-spec/bolt7/pyproject.toml + version = "1.0.2.186.post0"; + format = "pyproject"; inherit (clightning) src; + nativeBuildInputs = [ poetry-core ]; propagatedBuildInputs = [ pyln-proto ]; + checkInputs = [ pytestCheckHook ]; postUnpack = "sourceRoot=$sourceRoot/contrib/pyln-spec/bolt7"; - - # TODO-EXTERNAL: - # Remove when this fix is released - # https://github.com/ElementsProject/lightning/pull/4910 - postPatch = '' - sed -i 's|pyln.proto|pyln-proto|' requirements.txt - ''; } diff --git a/pkgs/python-packages/pyln-client/default.nix b/pkgs/python-packages/pyln-client/default.nix index 9ca49f8..b997ce4 100644 --- a/pkgs/python-packages/pyln-client/default.nix +++ b/pkgs/python-packages/pyln-client/default.nix @@ -1,14 +1,27 @@ -{ buildPythonPackage, clightning, pyln-bolt7, recommonmark, setuptools-scm }: +{ buildPythonPackage, poetry-core, pytestCheckHook, clightning, pyln-bolt7, pyln-proto }: buildPythonPackage rec { pname = "pyln-client"; version = clightning.version; + format = "pyproject"; inherit (clightning) src; - propagatedBuildInputs = [ pyln-bolt7 recommonmark setuptools-scm ]; + nativeBuildInputs = [ poetry-core ]; - SETUPTOOLS_SCM_PRETEND_VERSION = version; + propagatedBuildInputs = [ + pyln-bolt7 + pyln-proto + ]; + + checkInputs = [ pytestCheckHook ]; postUnpack = "sourceRoot=$sourceRoot/contrib/${pname}"; + + # Fix version typo in pyproject.toml + # TODO-EXTERNAL: + # This is already fixed upstream. Remove this after the next clightning release. + postPatch = '' + sed -i 's|pyln-bolt7 = "^1.0.186"|pyln-bolt7 = "^1.0.2.186"|' pyproject.toml + ''; } diff --git a/pkgs/python-packages/pyln-proto/default.nix b/pkgs/python-packages/pyln-proto/default.nix index 80303a2..29ee61a 100644 --- a/pkgs/python-packages/pyln-proto/default.nix +++ b/pkgs/python-packages/pyln-proto/default.nix @@ -1,38 +1,30 @@ -{ buildPythonPackage, clightning +{ buildPythonPackage +, clightning +, poetry-core +, pytestCheckHook , bitstring , cryptography , coincurve , base58 -, mypy -, pycparser -, setuptools-scm }: buildPythonPackage rec { pname = "pyln-proto"; version = clightning.version; + format = "pyproject"; inherit (clightning) src; + nativeBuildInputs = [ poetry-core ]; + propagatedBuildInputs = [ bitstring cryptography coincurve - pycparser base58 - mypy - setuptools-scm ]; - SETUPTOOLS_SCM_PRETEND_VERSION = version; + checkInputs = [ pytestCheckHook ]; postUnpack = "sourceRoot=$sourceRoot/contrib/pyln-proto"; - postPatch = '' - sed -i ' - s|pycparser==2.20|pycparser~=2.20| - s|coincurve ~= 13.0|coincurve == 15.0.0| - s|base58 ~= 2.0.1|base58 == 2.1.0| - s|mypy==0.790|mypy == 0.812| - ' requirements.txt - ''; } diff --git a/pkgs/python-packages/specific-versions/base58.nix b/pkgs/python-packages/specific-versions/base58.nix deleted file mode 100644 index 1437683..0000000 --- a/pkgs/python-packages/specific-versions/base58.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -, pyhamcrest -, pytest-benchmark -, pytestCheckHook -, pythonOlder -}: - -buildPythonPackage rec { - pname = "base58"; - version = "2.1.0"; - disabled = pythonOlder "3.5"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256-FxpUe0o8YeGuOAciSm967HXjZMQ5XnViZJ1zNXaAAaI="; - }; - - checkInputs = [ - pyhamcrest - pytest-benchmark - pytestCheckHook - ]; - - pythonImportsCheck = [ "base58" ]; - - meta = with lib; { - description = "Base58 and Base58Check implementation"; - homepage = "https://github.com/keis/base58"; - license = licenses.mit; - maintainers = with maintainers; [ nyanloutre ]; - }; -}