Merge fort-nix/nix-bitcoin#430: NixOS 21.11

2f0f1ec15a hardened-extended: define kernelParams as separate strings (Erik Arvstedt)
71e9864060 netns-isolation: add compatibility with 21.11 (Erik Arvstedt)
8f641756ce tests: add TODO-EXTERNAL (Erik Arvstedt)
b2f7a80f9f tests: add compatibility with 21.11 (Erik Arvstedt)
88fa33479f clightning-plugins: update prometheus-client version pin (nixbitcoin)
c9c2104009 klein: use from upstream, update jmclient version pin (nixbitcoin)
048b49edae pyln-proto: add copy of base58 2.1.0 (nixbitcoin)
b5984a603c joinmarket: 0.9.3 -> 0.9.4 (nixbitcoin)
8433933251 node2nix composition: add writeShellScript (nixbitcoin)
6b725157e1 clboss: sslSupport -> opensslSupport (nixbitcoin)
1a8e7d6348 clightning: re-enable seccomp filtering (nixbitcoin)
16f5aa0561 update to NixOS 21.11 (nixbitcoin)

Pull request description:

ACKs for top commit:
  erikarvstedt:
    ACK 2f0f1ec15a
  jonasnick:
    ACK 2f0f1ec15a

Tree-SHA512: 0685f94f0bc2ab64faa8679e2b579e6f2f1e96809ff00b61083b89bd7367ac990ddc1221698ee0e6729303c2a9a120cd460f32aba4f16ac5beeada8b70b6ea5b

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "flake-utils": {
       "locked": {
-        "lastModified": 1637014545,
+        "lastModified": 1638122382,
-        "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=",
+        "narHash": "sha256-sQzZzAbvKEqN9s0bzWuYmRaA03v40gaJ4+iL1LXjaeI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4",
+        "rev": "74f7e4319258e287b0f9cb95426c9853b282730b",
         "type": "github"
       },
       "original": {
@@ -17,27 +17,27 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1637973395,
+        "lastModified": 1638887115,
-        "narHash": "sha256-wp/QgsnnKJAIMZeQgJT1bWQ/OutZ+80lHwtpYAUWDfE=",
+        "narHash": "sha256-emjtIeqyJ84Eb3X7APJruTrwcfnHQKs55XGljj62prs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4f37689c8a219a9d756c5ff38525ad09349f422f",
+        "rev": "1bd4bbd49bef217a3d1adea43498270d6e779d65",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-21.05",
+        "ref": "nixos-21.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgsUnstable": {
       "locked": {
-        "lastModified": 1638036523,
+        "lastModified": 1638910453,
-        "narHash": "sha256-ZL6gogsuBmhBvIro+YwRKrypYhwVPCOOO7FmhOV/xyE=",
+        "narHash": "sha256-fofA4tsAAdNgE+9Py0VsVbyX2ZQkgV+CVqQKGBA/dLE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9c191ebcdfe917043195c54ab6ae8e934434fe7b",
+        "rev": "f225322e3bea8638304adfcf415cd11de99f2208",
         "type": "github"
       },
       "original": {

flake.nix

@@ -5,7 +5,7 @@
   '';
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.05";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-21.11";
     nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
   };

modules/clightning.nix

@@ -148,14 +148,6 @@ in {
         Restart = "on-failure";
         RestartSec = "10s";
         ReadWritePaths = cfg.dataDir;
-
-        # TODO-EXTERNAL:
-        # The seccomp version used by systemd in NixOS 21.05 doesn't support
-        # handling syscall 436 (close_range), which has only recently been added:
-        # https://github.com/seccomp/libseccomp/commit/ac849e7960547d418009a783da654d5917dbfe2d
-        #
-        # Disable seccomp filtering because clightning depends on this syscall.
-        SystemCallFilter = [];
       } // nbLib.allowedIPAddresses cfg.tor.enforce;
       # Wait until the rpc socket appears
       postStart = ''

modules/joinmarket.nix

@@ -155,6 +155,14 @@ let
     port = 6667
     usessl = false
     ${socks5Settings}
+
+    # ilita
+    [MESSAGING:server4]
+    host = ilitafrzzgxymv6umx2ux7kbz3imyeko6cnqkvy4nisjjj4qpqkrptid.onion
+    channel = joinmarket-pit
+    port = 6667
+    usessl = false
+    ${socks5Settings}
   '';
 
   # Based on https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/jmclient/jmclient/configure.py

modules/netns-isolation.nix

@@ -122,6 +122,7 @@ in {
       source = config.nix-bitcoin.pkgs.netns-exec;
       capabilities = "cap_sys_admin=ep";
       owner = cfg.allowedUser;
+      group = ""; # Set to the group of `owner`
       permissions = "550";
     };
 

modules/presets/hardened-extended.nix

@@ -93,7 +93,7 @@
 
     # These parameters prevent information leaks during boot and must be used
     # in combination with the kernel.printk
-    "quiet loglevel=0"
+    "quiet" "loglevel=0"
   ];
 
   boot.blacklistedKernelModules = [

pkgs/cl-rest/composition.nix

@@ -6,7 +6,7 @@
 
 let
   nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
-    inherit (pkgs) stdenv lib python2 runCommand writeTextFile;
+    inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
     inherit pkgs nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };

pkgs/clboss/default.nix

@@ -1,7 +1,7 @@
 { lib, stdenv, fetchurl, pkgconfig, curl, libev, sqlite }:
 
 let
-  curlWithGnuTLS = curl.override { gnutlsSupport = true; sslSupport = false; };
+  curlWithGnuTLS = curl.override { gnutlsSupport = true; opensslSupport = false; };
 in
 stdenv.mkDerivation rec {
   pname = "clboss";

pkgs/clightning-plugins/default.nix

@@ -18,7 +18,7 @@ let
     prometheus = {
       extraPkgs = [ prometheus_client ];
       patchRequirements =
-        "--replace prometheus-client==0.6.0 prometheus-client==0.9.0"
+        "--replace prometheus-client==0.6.0 prometheus-client==0.11.0"
         + " --replace pyln-client~=0.9.3 pyln-client~=0.10.1";
     };
     rebalance = {};

pkgs/joinmarket/default.nix

@@ -1,10 +1,10 @@
 { stdenv, lib, fetchurl, python3, nbPython3Packages, pkgs }:
 
 let
-  version = "0.9.3";
+  version = "0.9.4";
   src = fetchurl {
     url = "https://github.com/JoinMarket-Org/joinmarket-clientserver/archive/v${version}.tar.gz";
-    sha256 = "0j00jjqbppvcj52dpyjfqzwsm86xf9h2yf15j35ah5gsdr317dgq";
+    sha256 = "1xkz274g9lv5yif77h0mci1fsgam56sdc8m281q3a8hij9nmzmq1";
   };
 
   runtimePackages = with nbPython3Packages; [

pkgs/pinned.nix

@@ -2,12 +2,8 @@
 pkgs: pkgsUnstable:
 {
   inherit (pkgs)
-    lndconnect;
-
-  inherit (pkgsUnstable)
     bitcoin
     bitcoind
-    btcpayserver
     charge-lnd
     clightning
     electrs
@@ -16,6 +12,10 @@ pkgs: pkgsUnstable:
     hwi
     lightning-loop
     lightning-pool
+    lndconnect;
+
+  inherit (pkgsUnstable)
+    btcpayserver
     lnd
     nbxplorer;
 

pkgs/python-packages/default.nix

@@ -24,19 +24,16 @@ in {
 
   ## Specific versions of packages that already exist in nixpkgs
 
+  # base58 2.1.0, required by pyln-proto
+  base58 = callPackage ./specific-versions/base58.nix {};
+
   # cryptography 3.3.2, required by joinmarketdaemon
   cryptography = callPackage ./specific-versions/cryptography {};
   cryptography_vectors = callPackage ./specific-versions/cryptography/vectors.nix {};
 
-  # twisted 20.3.0, required by joinmarketbase
-  twisted = callPackage ./specific-versions/twisted.nix {};
-
   # autobahn 20.12.3, required by joinmarketclient
   autobahn = callPackage ./specific-versions/autobahn.nix {};
 
-  # klein 20.6.0, required by joinmarketclient
-  klein = callPackage ./specific-versions/klein.nix {};
-
   # tubes 0.2.0, required by klein
   tubes = callPackage ./specific-versions/tubes.nix {};
 

pkgs/python-packages/jmclient/default.nix

@@ -11,6 +11,11 @@ buildPythonPackage rec {
   # configparser may need to be compiled with python_version<"3.2"
   propagatedBuildInputs = [ future configparser joinmarketbase mnemonic argon2_cffi bencoderpyx pyaes klein pyjwt autobahn cryptography ];
 
+  patchPhase = ''
+    substituteInPlace setup.py \
+      --replace "'klein==20.6.0'" "'klein==21.8.0'"
+  '';
+
   meta = with lib; {
     description = "Client library for Bitcoin coinjoins";
     homepage = "https://github.com/Joinmarket-Org/joinmarket-clientserver";

pkgs/python-packages/specific-versions/base58.nix

@@ -0,0 +1,34 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pyhamcrest
+, pytest-benchmark
+, pytestCheckHook
+, pythonOlder
+}:
+
+buildPythonPackage rec {
+  pname = "base58";
+  version = "2.1.0";
+  disabled = pythonOlder "3.5";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-FxpUe0o8YeGuOAciSm967HXjZMQ5XnViZJ1zNXaAAaI=";
+  };
+
+  checkInputs = [
+    pyhamcrest
+    pytest-benchmark
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [ "base58" ];
+
+  meta = with lib; {
+    description = "Base58 and Base58Check implementation";
+    homepage = "https://github.com/keis/base58";
+    license = licenses.mit;
+    maintainers = with maintainers; [ nyanloutre ];
+  };
+}

pkgs/python-packages/specific-versions/klein.nix

@@ -1,29 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi, python
-, attrs, enum34, hyperlink, incremental, six, twisted, typing, tubes, werkzeug, zope_interface
-, hypothesis, treq
-}:
-
-buildPythonPackage rec {
-  pname = "klein";
-  version = "20.6.0";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "sha256-ZYS5zf9JWbnc7pWhwcIAEPUhoqEsT/PN2LkDqbDpk/Y=";
-  };
-
-  propagatedBuildInputs = [ attrs enum34 hyperlink incremental six twisted typing tubes werkzeug zope_interface ];
-
-  checkInputs = [ hypothesis treq ];
-
-  checkPhase = ''
-    ${python.interpreter} -m twisted.trial -j $NIX_BUILD_CORES klein
-  '';
-
-  meta = with lib; {
-    description = "Klein Web Micro-Framework";
-    homepage    = "https://github.com/twisted/klein";
-    license     = licenses.mit;
-    maintainers = with maintainers; [ exarkun ];
-  };
-}

pkgs/python-packages/specific-versions/twisted.nix

@@ -1,62 +0,0 @@
-{ lib, stdenv
-, buildPythonPackage
-, fetchPypi
-, python
-, zope_interface
-, incremental
-, automat
-, constantly
-, hyperlink
-, pyhamcrest
-, attrs
-, pyopenssl
-, service-identity
-, setuptools
-, idna
-}:
-buildPythonPackage rec {
-  pname = "Twisted";
-  version = "20.3.0";
-
-  src = fetchPypi {
-    inherit pname version;
-    extension = "tar.bz2";
-    sha256 = "040yzha6cyshnn6ljgk2birgh6mh2cnra48xp5ina5vfsnsmab6p";
-  };
-
-  propagatedBuildInputs = [ zope_interface incremental automat constantly hyperlink pyhamcrest attrs setuptools ];
-
-  passthru.extras.tls = [ pyopenssl service-identity idna ];
-
-  # Patch t.p._inotify to point to libc. Without this,
-  # twisted.python.runtime.platform.supportsINotify() == False
-  patchPhase = lib.optionalString stdenv.isLinux ''
-    substituteInPlace src/twisted/python/_inotify.py --replace \
-      "ctypes.util.find_library('c')" "'${stdenv.glibc.out}/lib/libc.so.6'"
-  '';
-
-  # Generate Twisted's plug-in cache.  Twisted users must do it as well.  See
-  # http://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3
-  # and http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=477103 for
-  # details.
-  postFixup = ''
-    $out/bin/twistd --help > /dev/null
-  '';
-
-  checkPhase = ''
-    ${python.interpreter} -m unittest discover -s twisted/test
-  '';
-  # Tests require network
-  doCheck = false;
-
-  meta = with lib; {
-    homepage = "https://twistedmatrix.com/";
-    description = "Twisted, an event-driven networking engine written in Python";
-    longDescription = ''
-      Twisted is an event-driven networking engine written in Python
-      and licensed under the MIT license.
-    '';
-    license = licenses.mit;
-    maintainers = [ ];
-  };
-}

pkgs/rtl/composition.nix

@@ -6,7 +6,7 @@
 
 let
   nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
-    inherit (pkgs) stdenv lib python2 runCommand writeTextFile;
+    inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
     inherit pkgs nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };

pkgs/spark-wallet/composition.nix

@@ -6,7 +6,7 @@
 
 let
   nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
-    inherit (pkgs) stdenv lib python2 runCommand writeTextFile;
+    inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
     inherit pkgs nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };

test/tests.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict
 import json
-
+import re
-logger = machine.logger
 
 def succeed(*cmds):
     """Returns the concatenated output of all cmds"""
@@ -34,7 +33,7 @@ def wait_for_open_port(address, port):
         status, _ = machine.execute(f"nc -z {address} {port}")
         return status == 0
 
-    with logger.nested(f"Waiting for TCP port {address}:{port}"):
+    with machine.nested(f"Waiting for TCP port {address}:{port}"):
         retry(is_port_open)
 
 
@@ -58,7 +57,7 @@ def run_tests():
         raise RuntimeError(f"The following tests are enabled but not defined: {enabled}")
     machine.connect()  # Visually separate boot output from the test output
     for test in to_run:
-        with logger.nested(f"test: {test}"):
+        with machine.nested(f"test: {test}"):
             tests[test]()
 
 def run_test(test):
@@ -139,9 +138,9 @@ def _():
                 f"Output of 'lightning-cli plugin list':\n{plugin_list}"
             )
         else:
-            logger.log("Active clightning plugins:")
+            machine.log("Active clightning plugins:")
             for p in test_data["clightning-plugins"]:
-                logger.log(os.path.basename(p))
+                machine.log(os.path.basename(p))
 
 @test("lnd")
 def _():
@@ -314,7 +313,7 @@ def _():
 
     succeed("systemctl stop bitcoind")
     succeed("systemctl start duplicity")
-    machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded."))
+    machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Deactivated successfully."))
     run_duplicity = "export $(cat /secrets/backup-encryption-env); duplicity"
     # Files in backup and /var/lib should be identical
     assert_matches(
@@ -401,6 +400,7 @@ def _():
         succeed("runuser -u operator -- pool orders list")
     if enabled("btcpayserver"):
         machine.wait_until_succeeds(log_has_string("nbxplorer", f"BTC: Starting scan at block {num_blocks}"))
+        # TODO-EXTERNAL:
         # nbxplorer 2.2.16 currently fails with with lbtc (liquidd) on regtest
         # LBTC: Full node version detected: 210000
         # LBTC: RPC connection successful