From 16f5aa0561265bb9af8dc31d4d09d32c3597ad9e Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 12:12:32 +0000 Subject: [PATCH 01/12] update to NixOS 21.11 --- flake.lock | 20 ++++++++++---------- flake.nix | 2 +- pkgs/pinned.nix | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/flake.lock b/flake.lock index 1bcfb60..3134c04 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flake-utils": { "locked": { - "lastModified": 1637014545, - "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=", + "lastModified": 1638122382, + "narHash": "sha256-sQzZzAbvKEqN9s0bzWuYmRaA03v40gaJ4+iL1LXjaeI=", "owner": "numtide", "repo": "flake-utils", - "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4", + "rev": "74f7e4319258e287b0f9cb95426c9853b282730b", "type": "github" }, "original": { @@ -17,27 +17,27 @@ }, "nixpkgs": { "locked": { - "lastModified": 1637973395, - "narHash": "sha256-wp/QgsnnKJAIMZeQgJT1bWQ/OutZ+80lHwtpYAUWDfE=", + "lastModified": 1638887115, + "narHash": "sha256-emjtIeqyJ84Eb3X7APJruTrwcfnHQKs55XGljj62prs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4f37689c8a219a9d756c5ff38525ad09349f422f", + "rev": "1bd4bbd49bef217a3d1adea43498270d6e779d65", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-21.05", + "ref": "nixos-21.11", "repo": "nixpkgs", "type": "github" } }, "nixpkgsUnstable": { "locked": { - "lastModified": 1638036523, - "narHash": "sha256-ZL6gogsuBmhBvIro+YwRKrypYhwVPCOOO7FmhOV/xyE=", + "lastModified": 1638910453, + "narHash": "sha256-fofA4tsAAdNgE+9Py0VsVbyX2ZQkgV+CVqQKGBA/dLE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9c191ebcdfe917043195c54ab6ae8e934434fe7b", + "rev": "f225322e3bea8638304adfcf415cd11de99f2208", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 302b612..34e635e 100644 --- a/flake.nix +++ b/flake.nix @@ -5,7 +5,7 @@ ''; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.11"; nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; }; diff --git a/pkgs/pinned.nix b/pkgs/pinned.nix index 90954f8..8ec11d1 100644 --- a/pkgs/pinned.nix +++ b/pkgs/pinned.nix @@ -2,12 +2,8 @@ pkgs: pkgsUnstable: { inherit (pkgs) - lndconnect; - - inherit (pkgsUnstable) bitcoin bitcoind - btcpayserver charge-lnd clightning electrs @@ -16,6 +12,10 @@ pkgs: pkgsUnstable: hwi lightning-loop lightning-pool + lndconnect; + + inherit (pkgsUnstable) + btcpayserver lnd nbxplorer; From 1a8e7d63480d635d96db70cf842384fbd2ac42aa Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 12:13:09 +0000 Subject: [PATCH 02/12] clightning: re-enable seccomp filtering --- modules/clightning.nix | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/clightning.nix b/modules/clightning.nix index 1c0cabf..4e1f3dc 100644 --- a/modules/clightning.nix +++ b/modules/clightning.nix @@ -148,14 +148,6 @@ in { Restart = "on-failure"; RestartSec = "10s"; ReadWritePaths = cfg.dataDir; - - # TODO-EXTERNAL: - # The seccomp version used by systemd in NixOS 21.05 doesn't support - # handling syscall 436 (close_range), which has only recently been added: - # https://github.com/seccomp/libseccomp/commit/ac849e7960547d418009a783da654d5917dbfe2d - # - # Disable seccomp filtering because clightning depends on this syscall. - SystemCallFilter = []; } // nbLib.allowedIPAddresses cfg.tor.enforce; # Wait until the rpc socket appears postStart = '' From 6b725157e1d6f16403c888647d8302a3daba5ac9 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 12:14:34 +0000 Subject: [PATCH 03/12] clboss: sslSupport -> opensslSupport --- pkgs/clboss/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clboss/default.nix b/pkgs/clboss/default.nix index b45137c..7c89db9 100644 --- a/pkgs/clboss/default.nix +++ b/pkgs/clboss/default.nix @@ -1,7 +1,7 @@ { lib, stdenv, fetchurl, pkgconfig, curl, libev, sqlite }: let - curlWithGnuTLS = curl.override { gnutlsSupport = true; sslSupport = false; }; + curlWithGnuTLS = curl.override { gnutlsSupport = true; opensslSupport = false; }; in stdenv.mkDerivation rec { pname = "clboss"; From 84339332514559b4e83dd66f9c202fadedcd2ce7 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 12:15:23 +0000 Subject: [PATCH 04/12] node2nix composition: add writeShellScript --- pkgs/cl-rest/composition.nix | 2 +- pkgs/rtl/composition.nix | 2 +- pkgs/spark-wallet/composition.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/cl-rest/composition.nix b/pkgs/cl-rest/composition.nix index 3eb1de2..f49b28e 100644 --- a/pkgs/cl-rest/composition.nix +++ b/pkgs/cl-rest/composition.nix @@ -6,7 +6,7 @@ let nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" { - inherit (pkgs) stdenv lib python2 runCommand writeTextFile; + inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript; inherit pkgs nodejs; libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; }; diff --git a/pkgs/rtl/composition.nix b/pkgs/rtl/composition.nix index 3eb1de2..f49b28e 100644 --- a/pkgs/rtl/composition.nix +++ b/pkgs/rtl/composition.nix @@ -6,7 +6,7 @@ let nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" { - inherit (pkgs) stdenv lib python2 runCommand writeTextFile; + inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript; inherit pkgs nodejs; libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; }; diff --git a/pkgs/spark-wallet/composition.nix b/pkgs/spark-wallet/composition.nix index 3eb1de2..f49b28e 100644 --- a/pkgs/spark-wallet/composition.nix +++ b/pkgs/spark-wallet/composition.nix @@ -6,7 +6,7 @@ let nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" { - inherit (pkgs) stdenv lib python2 runCommand writeTextFile; + inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript; inherit pkgs nodejs; libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; }; From b5984a603c746050c5839ded3c5104e91a8fa0c1 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 12:16:14 +0000 Subject: [PATCH 05/12] joinmarket: 0.9.3 -> 0.9.4 Ilita IRC server & use upstream twisted again --- modules/joinmarket.nix | 8 +++ pkgs/joinmarket/default.nix | 4 +- pkgs/python-packages/default.nix | 3 - .../specific-versions/twisted.nix | 62 ------------------- 4 files changed, 10 insertions(+), 67 deletions(-) delete mode 100644 pkgs/python-packages/specific-versions/twisted.nix diff --git a/modules/joinmarket.nix b/modules/joinmarket.nix index 94a3479..79565ad 100644 --- a/modules/joinmarket.nix +++ b/modules/joinmarket.nix @@ -155,6 +155,14 @@ let port = 6667 usessl = false ${socks5Settings} + + # ilita + [MESSAGING:server4] + host = ilitafrzzgxymv6umx2ux7kbz3imyeko6cnqkvy4nisjjj4qpqkrptid.onion + channel = joinmarket-pit + port = 6667 + usessl = false + ${socks5Settings} ''; # Based on https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/jmclient/jmclient/configure.py diff --git a/pkgs/joinmarket/default.nix b/pkgs/joinmarket/default.nix index 898375a..931854d 100644 --- a/pkgs/joinmarket/default.nix +++ b/pkgs/joinmarket/default.nix @@ -1,10 +1,10 @@ { stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }: let - version = "0.9.3"; + version = "0.9.4"; src = fetchurl { url = "https://github.com/JoinMarket-Org/joinmarket-clientserver/archive/v${version}.tar.gz"; - sha256 = "0j00jjqbppvcj52dpyjfqzwsm86xf9h2yf15j35ah5gsdr317dgq"; + sha256 = "1xkz274g9lv5yif77h0mci1fsgam56sdc8m281q3a8hij9nmzmq1"; }; runtimePackages = with nbPython3Packages; [ diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index 44d0001..023e64e 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -28,9 +28,6 @@ in { cryptography = callPackage ./specific-versions/cryptography {}; cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {}; - # twisted 20.3.0, required by joinmarketbase - twisted = callPackage ./specific-versions/twisted.nix {}; - # autobahn 20.12.3, required by joinmarketclient autobahn = callPackage ./specific-versions/autobahn.nix {}; diff --git a/pkgs/python-packages/specific-versions/twisted.nix b/pkgs/python-packages/specific-versions/twisted.nix deleted file mode 100644 index 4215657..0000000 --- a/pkgs/python-packages/specific-versions/twisted.nix +++ /dev/null @@ -1,62 +0,0 @@ -{ lib, stdenv -, buildPythonPackage -, fetchPypi -, python -, zope_interface -, incremental -, automat -, constantly -, hyperlink -, pyhamcrest -, attrs -, pyopenssl -, service-identity -, setuptools -, idna -}: -buildPythonPackage rec { - pname = "Twisted"; - version = "20.3.0"; - - src = fetchPypi { - inherit pname version; - extension = "tar.bz2"; - sha256 = "040yzha6cyshnn6ljgk2birgh6mh2cnra48xp5ina5vfsnsmab6p"; - }; - - propagatedBuildInputs = [ zope_interface incremental automat constantly hyperlink pyhamcrest attrs setuptools ]; - - passthru.extras.tls = [ pyopenssl service-identity idna ]; - - # Patch t.p._inotify to point to libc. Without this, - # twisted.python.runtime.platform.supportsINotify() == False - patchPhase = lib.optionalString stdenv.isLinux '' - substituteInPlace src/twisted/python/_inotify.py --replace \ - "ctypes.util.find_library('c')" "'${stdenv.glibc.out}/lib/libc.so.6'" - ''; - - # Generate Twisted's plug-in cache. Twisted users must do it as well. See - # http://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3 - # and http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=477103 for - # details. - postFixup = '' - $out/bin/twistd --help > /dev/null - ''; - - checkPhase = '' - ${python.interpreter} -m unittest discover -s twisted/test - ''; - # Tests require network - doCheck = false; - - meta = with lib; { - homepage = "https://twistedmatrix.com/"; - description = "Twisted, an event-driven networking engine written in Python"; - longDescription = '' - Twisted is an event-driven networking engine written in Python - and licensed under the MIT license. - ''; - license = licenses.mit; - maintainers = [ ]; - }; -} From 048b49edaef0fcd8275c1076f4e5957b53203983 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 12:36:14 +0000 Subject: [PATCH 06/12] pyln-proto: add copy of base58 2.1.0 --- pkgs/python-packages/default.nix | 3 ++ .../specific-versions/base58.nix | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 pkgs/python-packages/specific-versions/base58.nix diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index 023e64e..beb4551 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -24,6 +24,9 @@ in { ## Specific versions of packages that already exist in nixpkgs + # base58 2.1.0, required by pyln-proto + base58 = callPackage ./specific-versions/base58.nix {}; + # cryptography 3.3.2, required by joinmarketdaemon cryptography = callPackage ./specific-versions/cryptography {}; cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {}; diff --git a/pkgs/python-packages/specific-versions/base58.nix b/pkgs/python-packages/specific-versions/base58.nix new file mode 100644 index 0000000..1437683 --- /dev/null +++ b/pkgs/python-packages/specific-versions/base58.nix @@ -0,0 +1,34 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pyhamcrest +, pytest-benchmark +, pytestCheckHook +, pythonOlder +}: + +buildPythonPackage rec { + pname = "base58"; + version = "2.1.0"; + disabled = pythonOlder "3.5"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-FxpUe0o8YeGuOAciSm967HXjZMQ5XnViZJ1zNXaAAaI="; + }; + + checkInputs = [ + pyhamcrest + pytest-benchmark + pytestCheckHook + ]; + + pythonImportsCheck = [ "base58" ]; + + meta = with lib; { + description = "Base58 and Base58Check implementation"; + homepage = "https://github.com/keis/base58"; + license = licenses.mit; + maintainers = with maintainers; [ nyanloutre ]; + }; +} From c9c21040091c30e1f20ce0a83481be023194fc2c Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 13:23:23 +0000 Subject: [PATCH 07/12] klein: use from upstream, update jmclient version pin klein 20.6.0 was failing Twisted 21.7.0 tests. --- pkgs/python-packages/default.nix | 3 -- pkgs/python-packages/jmclient/default.nix | 5 ++++ .../specific-versions/klein.nix | 29 ------------------- 3 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 pkgs/python-packages/specific-versions/klein.nix diff --git a/pkgs/python-packages/default.nix b/pkgs/python-packages/default.nix index beb4551..0c0f9d3 100644 --- a/pkgs/python-packages/default.nix +++ b/pkgs/python-packages/default.nix @@ -34,9 +34,6 @@ in { # autobahn 20.12.3, required by joinmarketclient autobahn = callPackage ./specific-versions/autobahn.nix {}; - # klein 20.6.0, required by joinmarketclient - klein = callPackage ./specific-versions/klein.nix {}; - # tubes 0.2.0, required by klein tubes = callPackage ./specific-versions/tubes.nix {}; diff --git a/pkgs/python-packages/jmclient/default.nix b/pkgs/python-packages/jmclient/default.nix index 3a73dd4..6b9d8ef 100644 --- a/pkgs/python-packages/jmclient/default.nix +++ b/pkgs/python-packages/jmclient/default.nix @@ -11,6 +11,11 @@ buildPythonPackage rec { # configparser may need to be compiled with python_version<"3.2" propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes klein pyjwt autobahn cryptography ]; + patchPhase = '' + substituteInPlace setup.py \ + --replace "'klein==20.6.0'" "'klein==21.8.0'" + ''; + meta = with lib; { description = "Client library for Bitcoin coinjoins"; homepage = "https://github.com/Joinmarket-Org/joinmarket-clientserver"; diff --git a/pkgs/python-packages/specific-versions/klein.nix b/pkgs/python-packages/specific-versions/klein.nix deleted file mode 100644 index 4b5be48..0000000 --- a/pkgs/python-packages/specific-versions/klein.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, python -, attrs, enum34, hyperlink, incremental, six, twisted, typing, tubes, werkzeug, zope_interface -, hypothesis, treq -}: - -buildPythonPackage rec { - pname = "klein"; - version = "20.6.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256-ZYS5zf9JWbnc7pWhwcIAEPUhoqEsT/PN2LkDqbDpk/Y="; - }; - - propagatedBuildInputs = [ attrs enum34 hyperlink incremental six twisted typing tubes werkzeug zope_interface ]; - - checkInputs = [ hypothesis treq ]; - - checkPhase = '' - ${python.interpreter} -m twisted.trial -j $NIX_BUILD_CORES klein - ''; - - meta = with lib; { - description = "Klein Web Micro-Framework"; - homepage = "https://github.com/twisted/klein"; - license = licenses.mit; - maintainers = with maintainers; [ exarkun ]; - }; -} From 88fa33479fd9db0a66d056eab7aeefcd4ad67f21 Mon Sep 17 00:00:00 2001 From: nixbitcoin Date: Wed, 8 Dec 2021 13:26:03 +0000 Subject: [PATCH 08/12] clightning-plugins: update prometheus-client version pin --- pkgs/clightning-plugins/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clightning-plugins/default.nix b/pkgs/clightning-plugins/default.nix index daec96c..42531e9 100644 --- a/pkgs/clightning-plugins/default.nix +++ b/pkgs/clightning-plugins/default.nix @@ -18,7 +18,7 @@ let prometheus = { extraPkgs = [ prometheus_client ]; patchRequirements = - "--replace prometheus-client==0.6.0 prometheus-client==0.9.0" + "--replace prometheus-client==0.6.0 prometheus-client==0.11.0" + " --replace pyln-client~=0.9.3 pyln-client~=0.10.1"; }; rebalance = {}; From b2f7a80f9ff6dcfc1a4adb654b1be53f097b294f Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 9 Dec 2021 21:04:11 +0100 Subject: [PATCH 09/12] tests: add compatibility with 21.11 Adapt to the new test driver and systemd versions. --- test/tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/tests.py b/test/tests.py index d7224c4..ef61152 100644 --- a/test/tests.py +++ b/test/tests.py @@ -1,7 +1,6 @@ from collections import OrderedDict import json - -logger = machine.logger +import re def succeed(*cmds): """Returns the concatenated output of all cmds""" @@ -34,7 +33,7 @@ def wait_for_open_port(address, port): status, _ = machine.execute(f"nc -z {address} {port}") return status == 0 - with logger.nested(f"Waiting for TCP port {address}:{port}"): + with machine.nested(f"Waiting for TCP port {address}:{port}"): retry(is_port_open) @@ -58,7 +57,7 @@ def run_tests(): raise RuntimeError(f"The following tests are enabled but not defined: {enabled}") machine.connect() # Visually separate boot output from the test output for test in to_run: - with logger.nested(f"test: {test}"): + with machine.nested(f"test: {test}"): tests[test]() def run_test(test): @@ -139,9 +138,9 @@ def _(): f"Output of 'lightning-cli plugin list':\n{plugin_list}" ) else: - logger.log("Active clightning plugins:") + machine.log("Active clightning plugins:") for p in test_data["clightning-plugins"]: - logger.log(os.path.basename(p)) + machine.log(os.path.basename(p)) @test("lnd") def _(): @@ -314,7 +313,7 @@ def _(): succeed("systemctl stop bitcoind") succeed("systemctl start duplicity") - machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded.")) + machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Deactivated successfully.")) run_duplicity = "export $(cat /secrets/backup-encryption-env); duplicity" # Files in backup and /var/lib should be identical assert_matches( From 8f641756cef23015f9e2268fd5ff50bf476c2109 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 9 Dec 2021 21:04:12 +0100 Subject: [PATCH 10/12] tests: add TODO-EXTERNAL --- test/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/tests.py b/test/tests.py index ef61152..4250198 100644 --- a/test/tests.py +++ b/test/tests.py @@ -400,6 +400,7 @@ def _(): succeed("runuser -u operator -- pool orders list") if enabled("btcpayserver"): machine.wait_until_succeeds(log_has_string("nbxplorer", f"BTC: Starting scan at block {num_blocks}")) + # TODO-EXTERNAL: # nbxplorer 2.2.16 currently fails with with lbtc (liquidd) on regtest # LBTC: Full node version detected: 210000 # LBTC: RPC connection successful From 71e98640600319df3c8e024b72dd682d09622f26 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 9 Dec 2021 21:04:13 +0100 Subject: [PATCH 11/12] netns-isolation: add compatibility with 21.11 `security.wrappers` now requires `group` be defined. --- modules/netns-isolation.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/netns-isolation.nix b/modules/netns-isolation.nix index 471716a..47a837c 100644 --- a/modules/netns-isolation.nix +++ b/modules/netns-isolation.nix @@ -122,6 +122,7 @@ in { source = config.nix-bitcoin.pkgs.netns-exec; capabilities = "cap_sys_admin=ep"; owner = cfg.allowedUser; + group = ""; # Set to the group of `owner` permissions = "550"; }; From 2f0f1ec15aa5010864584fcaf828868b56421687 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 9 Dec 2021 21:04:14 +0100 Subject: [PATCH 12/12] hardened-extended: define kernelParams as separate strings This is now checked by the modules. --- modules/presets/hardened-extended.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/presets/hardened-extended.nix b/modules/presets/hardened-extended.nix index c091222..e9093b6 100644 --- a/modules/presets/hardened-extended.nix +++ b/modules/presets/hardened-extended.nix @@ -93,7 +93,7 @@ # These parameters prevent information leaks during boot and must be used # in combination with the kernel.printk - "quiet loglevel=0" + "quiet" "loglevel=0" ]; boot.blacklistedKernelModules = [