From 721ba1aeba6500ecf7bf109d11307fd396a6f8f8 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 24 Oct 2021 21:08:58 +0200 Subject: [PATCH 1/3] python-packages: separate `specific-versions` pkgs This simplifies maintenance. --- pkgs/python-packages/default.nix | 20 +++++++++++-------- .../cryptography/default.nix | 0 .../cryptography/vectors.nix | 0 .../recommonmark.nix} | 0 .../twisted.nix} | 0 5 files changed, 12 insertions(+), 8 deletions(-) rename pkgs/python-packages/{ => specific-versions}/cryptography/default.nix (100%) rename pkgs/python-packages/{ => specific-versions}/cryptography/vectors.nix (100%) rename pkgs/python-packages/{recommonmark/default.nix => specific-versions/recommonmark.nix} (100%) rename pkgs/python-packages/{twisted/default.nix => specific-versions/twisted.nix} (100%) diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index bb5cfdd..4d6d568 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -11,14 +11,6 @@ in { urldecode = callPackage ./urldecode {}; chromalog = callPackage ./chromalog {}; txzmq = callPackage ./txzmq {}; - recommonmark = callPackage ./recommonmark { inherit (super) recommonmark; }; - - # cryptography 3.3.2, required by joinmarketdaemon - cryptography = callPackage ./cryptography {}; - cryptography_vectors = callPackage ./cryptography/vectors.nix {}; - - # twisted 20.3.0, required by joinmarketbase - twisted = callPackage ./twisted {}; joinmarketbase = joinmarketPkg ./jmbase; joinmarketclient = joinmarketPkg ./jmclient; @@ -28,4 +20,16 @@ in { pyln-client = clightningPkg ./pyln-client; pyln-proto = clightningPkg ./pyln-proto; pylightning = clightningPkg ./pylightning; + + ## Specific versions of packages that already exist in nixpkgs + + # cryptography 3.3.2, required by joinmarketdaemon + cryptography = callPackage ./specific-versions/cryptography {}; + cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {}; + + # twisted 20.3.0, required by joinmarketbase + twisted = callPackage ./specific-versions/twisted.nix {}; + + # recommonmark 0.7.1, required by pyln-client + recommonmark = callPackage ./specific-versions/recommonmark.nix { inherit (super) recommonmark; }; } diff --git a/pkgs/python-packages/cryptography/default.nix b/pkgs/python-packages/specific-versions/cryptography/default.nix similarity index 100% rename from pkgs/python-packages/cryptography/default.nix rename to pkgs/python-packages/specific-versions/cryptography/default.nix diff --git a/pkgs/python-packages/cryptography/vectors.nix b/pkgs/python-packages/specific-versions/cryptography/vectors.nix similarity index 100% rename from pkgs/python-packages/cryptography/vectors.nix rename to pkgs/python-packages/specific-versions/cryptography/vectors.nix diff --git a/pkgs/python-packages/recommonmark/default.nix b/pkgs/python-packages/specific-versions/recommonmark.nix similarity index 100% rename from pkgs/python-packages/recommonmark/default.nix rename to pkgs/python-packages/specific-versions/recommonmark.nix diff --git a/pkgs/python-packages/twisted/default.nix b/pkgs/python-packages/specific-versions/twisted.nix similarity index 100% rename from pkgs/python-packages/twisted/default.nix rename to pkgs/python-packages/specific-versions/twisted.nix From a10aa21c695eb8ae482937b45ed4b545eabe765d Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Sun, 24 Oct 2021 21:14:09 +0200 Subject: [PATCH 2/3] joinmarket: 0.9.2 -> 0.9.3 --- .cirrus.yml | 7 ++-- modules/joinmarket.nix | 25 +++++++++++-- modules/netns-isolation.nix | 5 ++- pkgs/joinmarket/default.nix | 4 +-- pkgs/python-packages/default.nix | 9 +++++ pkgs/python-packages/jmclient/default.nix | 4 +-- .../specific-versions/autobahn.nix | 36 +++++++++++++++++++ .../specific-versions/klein.nix | 29 +++++++++++++++ .../specific-versions/tubes.nix | 29 +++++++++++++++ test/pkgs-unstable.nix | 3 +- 10 files changed, 139 insertions(+), 12 deletions(-) create mode 100644 pkgs/python-packages/specific-versions/autobahn.nix create mode 100644 pkgs/python-packages/specific-versions/klein.nix create mode 100644 pkgs/python-packages/specific-versions/tubes.nix diff --git a/.cirrus.yml b/.cirrus.yml index 5f82269..d679c0b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -31,9 +31,10 @@ task: - environment: scenario: netnsRegtest - - name: pkgs_unstable - environment: - nixpkgs: nixpkgs-unstable + # Disabled because this test is currently empty + # - name: pkgs_unstable + # environment: + # nixpkgs: nixpkgs-unstable # This script is run as root build_script: diff --git a/modules/joinmarket.nix b/modules/joinmarket.nix index 546b234..4801e4d 100644 --- a/modules/joinmarket.nix +++ b/modules/joinmarket.nix @@ -4,6 +4,22 @@ with lib; let options.services.joinmarket = { enable = mkEnableOption "JoinMarket"; + payjoinAddress = mkOption { + type = types.str; + default = "127.0.0.1"; + description = '' + The address where payjoin onion connections are forwarded to. + This address is never used directly, it only serves as the internal endpoint + for the payjoin onion service. + The onion service is automatically setup by joinmarket and accepts + connections at port 80. + ''; + }; + payjoinPort = mkOption { + type = types.port; + default = 64180; # A random private port + description = "The port corresponding to option `payjoinAddress`."; + }; dataDir = mkOption { type = types.path; default = "/var/lib/joinmarket"; @@ -78,7 +94,7 @@ let The average transaction fee you're adding to coinjoin transactions ''; }; - txfee_factor = mkOption { + txfee_contribution_factor = mkOption { type = types.float; default = 0.3; description = '' @@ -171,6 +187,7 @@ let native = true merge_algorithm = default tx_fees = 3 + tx_fees_factor = 0.2 absurd_fee_per_kb = 350000 max_sweep_fee_change = 0.8 tx_broadcast = self @@ -192,6 +209,8 @@ let onion_socks5_host = ${torAddress.addr} onion_socks5_port = ${toString torAddress.port} tor_control_host = unix:/run/tor/control + onion_serving_host = ${cfg.payjoinAddress} + onion_serving_port = ${toString cfg.payjoinPort} hidden_service_ssl = false [YIELDGENERATOR] @@ -199,8 +218,8 @@ let cjfee_a = ${toString yg.cjfee_a} cjfee_r = ${toString yg.cjfee_r} cjfee_factor = ${toString yg.cjfee_factor} - txfee = ${toString yg.txfee} - txfee_factor = ${toString yg.txfee_factor} + txfee_contribution = 0 + txfee_contribution_factor = ${toString yg.txfee_contribution_factor} minsize = ${toString yg.minsize} size_factor = ${toString yg.size_factor} gaplimit = 6 diff --git a/modules/netns-isolation.nix b/modules/netns-isolation.nix index dfa44d8..4b612f5 100644 --- a/modules/netns-isolation.nix +++ b/modules/netns-isolation.nix @@ -313,7 +313,10 @@ in { services.nbxplorer.address = netns.nbxplorer.address; services.btcpayserver.address = netns.btcpayserver.address; - services.joinmarket.cliExec = mkCliExec "joinmarket"; + services.joinmarket = { + payjoinAddress = netns.joinmarket.address; + cliExec = mkCliExec "joinmarket"; + }; systemd.services.joinmarket-yieldgenerator.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-joinmarket"; services.joinmarket-ob-watcher.address = netns.joinmarket-ob-watcher.address; diff --git a/pkgs/joinmarket/default.nix b/pkgs/joinmarket/default.nix index 5d76944..898375a 100644 --- a/pkgs/joinmarket/default.nix +++ b/pkgs/joinmarket/default.nix @@ -1,10 +1,10 @@ { stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }: let - version = "0.9.2"; + version = "0.9.3"; src = fetchurl { url = "https://github.com/JoinMarket-Org/joinmarket-clientserver/archive/v${version}.tar.gz"; - sha256 = "0nzhnf2324d0qc9pljcjwpzpvhcclqg0ijvzb1skmn73s4f25akg"; + sha256 = "0j00jjqbppvcj52dpyjfqzwsm86xf9h2yf15j35ah5gsdr317dgq"; }; runtimePackages = with nbPython3Packages; [ diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index 4d6d568..ea6d36a 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -30,6 +30,15 @@ in { # twisted 20.3.0, required by joinmarketbase twisted = callPackage ./specific-versions/twisted.nix {}; + # autobahn 20.12.3, required by joinmarketclient + autobahn = callPackage ./specific-versions/autobahn.nix {}; + + # klein 20.6.0, required by joinmarketclient + klein = callPackage ./specific-versions/klein.nix {}; + + # tubes 0.2.0, required by klein + tubes = callPackage ./specific-versions/tubes.nix {}; + # recommonmark 0.7.1, required by pyln-client recommonmark = callPackage ./specific-versions/recommonmark.nix { inherit (super) recommonmark; }; } diff --git a/pkgs/python-packages/jmclient/default.nix b/pkgs/python-packages/jmclient/default.nix index cf8702d..3a73dd4 100644 --- a/pkgs/python-packages/jmclient/default.nix +++ b/pkgs/python-packages/jmclient/default.nix @@ -1,4 +1,4 @@ -{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon }: +{ version, src, lib, buildPythonPackage, fetchurl, future, configparser, joinmarketbase, joinmarketdaemon, mnemonic, argon2_cffi, bencoderpyx, pyaes, joinmarketbitcoin, txtorcon, klein, pyjwt, autobahn, cryptography }: buildPythonPackage rec { pname = "joinmarketclient"; @@ -9,7 +9,7 @@ buildPythonPackage rec { checkInputs = [ joinmarketbitcoin joinmarketdaemon txtorcon ]; # configparser may need to be compiled with python_version<"3.2" - propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes ]; + propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes klein pyjwt autobahn cryptography ]; meta = with lib; { description = "Client library for Bitcoin coinjoins"; diff --git a/pkgs/python-packages/specific-versions/autobahn.nix b/pkgs/python-packages/specific-versions/autobahn.nix new file mode 100644 index 0000000..2557100 --- /dev/null +++ b/pkgs/python-packages/specific-versions/autobahn.nix @@ -0,0 +1,36 @@ +# Like nixpkgs revision 8d668463b0883b2e21ba2e2635cd5f9bbc409b18 +# but without Python 2 support + +{ lib, buildPythonPackage, fetchPypi, + six, txaio, twisted, zope_interface, cffi, + mock, pytest, cryptography, pynacl +}: +buildPythonPackage rec { + pname = "autobahn"; + version = "20.12.3"; + + src = fetchPypi { + inherit pname version; + sha256 = "15b8zm7jalwisfwc08szxy3bh2bnn0hd41dbsnswi0lqwbh962j1"; + }; + + propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ]; + + checkInputs = [ mock pytest ]; + checkPhase = '' + runHook preCheck + USE_TWISTED=true py.test $out + runHook postCheck + ''; + + # Tests do no seem to be compatible yet with pytest 5.1 + # https://github.com/crossbario/autobahn-python/issues/1235 + doCheck = false; + + meta = with lib; { + description = "WebSocket and WAMP in Python for Twisted and asyncio."; + homepage = "https://crossbar.io/autobahn"; + license = licenses.mit; + maintainers = with maintainers; [ nand0p ]; + }; +} diff --git a/pkgs/python-packages/specific-versions/klein.nix b/pkgs/python-packages/specific-versions/klein.nix new file mode 100644 index 0000000..4b5be48 --- /dev/null +++ b/pkgs/python-packages/specific-versions/klein.nix @@ -0,0 +1,29 @@ +{ lib, buildPythonPackage, fetchPypi, python +, attrs, enum34, hyperlink, incremental, six, twisted, typing, tubes, werkzeug, zope_interface +, hypothesis, treq +}: + +buildPythonPackage rec { + pname = "klein"; + version = "20.6.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-ZYS5zf9JWbnc7pWhwcIAEPUhoqEsT/PN2LkDqbDpk/Y="; + }; + + propagatedBuildInputs = [ attrs enum34 hyperlink incremental six twisted typing tubes werkzeug zope_interface ]; + + checkInputs = [ hypothesis treq ]; + + checkPhase = '' + ${python.interpreter} -m twisted.trial -j $NIX_BUILD_CORES klein + ''; + + meta = with lib; { + description = "Klein Web Micro-Framework"; + homepage = "https://github.com/twisted/klein"; + license = licenses.mit; + maintainers = with maintainers; [ exarkun ]; + }; +} diff --git a/pkgs/python-packages/specific-versions/tubes.nix b/pkgs/python-packages/specific-versions/tubes.nix new file mode 100644 index 0000000..0dbfe22 --- /dev/null +++ b/pkgs/python-packages/specific-versions/tubes.nix @@ -0,0 +1,29 @@ +{ lib, buildPythonPackage, fetchPypi, python +, characteristic, six, twisted +}: + +buildPythonPackage rec { + pname = "tubes"; + version = "0.2.0"; + + src = fetchPypi { + pname = "Tubes"; + inherit version; + sha256 = "0sg1gg2002h1xsgxigznr1zk1skwmhss72dzk6iysb9k9kdgymcd"; + }; + + propagatedBuildInputs = [ characteristic six twisted ]; + + checkPhase = '' + ${python.interpreter} -m twisted.trial -j $NIX_BUILD_CORES tubes + ''; + + pythonImportsCheck = [ "tubes" ]; + + meta = with lib; { + description = "a data-processing and flow-control engine for event-driven programs"; + homepage = "https://github.com/twisted/tubes"; + license = licenses.mit; + maintainers = with maintainers; [ exarkun ]; + }; +} diff --git a/test/pkgs-unstable.nix b/test/pkgs-unstable.nix index 444eb4d..b093fff 100644 --- a/test/pkgs-unstable.nix +++ b/test/pkgs-unstable.nix @@ -3,7 +3,8 @@ let pkgs = import pinned.nixpkgs-unstable {}; nbPkgs = import ../pkgs { inherit pkgs; }; pkgsUnstable = with nbPkgs; [ - joinmarket + # Disabled because joinmarket dependencies currently don't build on on unstable. + # joinmarket ]; in pkgs.writeText "pkgs-unstable" (pkgs.lib.concatMapStringsSep "\n" toString pkgsUnstable) From d5ce1c43a81869dacb25c0c9fd537ad226afd547 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Mon, 25 Oct 2021 13:30:53 +0000 Subject: [PATCH 3/3] test: make joinmarket work with regtest --- modules/joinmarket.nix | 11 ++++++----- test/tests.nix | 3 --- test/tests.py | 9 ++++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/joinmarket.nix b/modules/joinmarket.nix index 4801e4d..58ca77a 100644 --- a/modules/joinmarket.nix +++ b/modules/joinmarket.nix @@ -169,8 +169,8 @@ let use_ssl = false [BLOCKCHAIN] - blockchain_source = bitcoin-rpc - network = ${bitcoind.network} + blockchain_source = ${bitcoind.makeNetworkName "bitcoin-rpc" "regtest"} + network = ${bitcoind.makeNetworkName "mainnet" "testnet"} rpc_host = ${nbLib.address bitcoind.rpc.address} rpc_port = ${toString bitcoind.rpc.port} rpc_user = ${bitcoind.rpc.users.privileged.name} @@ -282,15 +282,16 @@ in { echo "rpc_password = $(cat ${secretsDir}/bitcoin-rpcpassword-privileged)" } > '${cfg.dataDir}/joinmarket.cfg' ''; - # Generating wallets (jmclient/wallet.py) is only supported for mainnet or testnet - postStart = mkIf (bitcoind.network == "mainnet") '' + postStart = '' walletname=wallet.jmdat wallet=${cfg.dataDir}/wallets/$walletname if [[ ! -f $wallet ]]; then ${optionalString (cfg.rpcWalletFile != null) '' echo "Create watch-only wallet ${cfg.rpcWalletFile}" if ! output=$(${bitcoind.cli}/bin/bitcoin-cli -named createwallet \ - wallet_name="${cfg.rpcWalletFile}" disable_private_keys=true 2>&1); then + wallet_name="${cfg.rpcWalletFile}" \ + ${optionalString (!bitcoind.regtest) "disable_private_keys=true"} 2>&1 + ); then # Ignore error if bitcoind wallet already exists if [[ $output != *"already exists"* ]]; then echo "$output" diff --git a/test/tests.nix b/test/tests.nix index 6f5d741..6e0710f 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -248,9 +248,6 @@ let services.lightning-pool.extraConfig = '' auctionserver=localhost ''; - - # Needs wallet support which is unavailable for regtest - services.joinmarket.yieldgenerator.enable = mkForce false; }; ## Examples / debug helper diff --git a/test/tests.py b/test/tests.py index e8a9534..163e147 100644 --- a/test/tests.py +++ b/test/tests.py @@ -218,9 +218,12 @@ def _(): @test("joinmarket-yieldgenerator") def _(): - machine.wait_until_succeeds( - log_has_string("joinmarket-yieldgenerator", "Critical error updating blockheight.") - ) + if "regtest" in enabled_tests: + expected_log_msg = "You do not have the minimum required amount of coins to be a maker" + else: + expected_log_msg = "Critical error updating blockheight." + + machine.wait_until_succeeds(log_has_string("joinmarket-yieldgenerator", expected_log_msg)) @test("joinmarket-ob-watcher") def _():