Merge fort-nix/nix-bitcoin#538: rtl: 0.12.3 -> 0.13.0, clightning-rest: 0.7.2 -> 0.8.0

53dd2a1ae2 cl-rest: 0.7.2 -> 0.8.0 (Erik Arvstedt)
617ed4c8e8 rtl: 0.12.3-beta -> 0.13.0 (Erik Arvstedt)
e63dafe0f7 pkgs: add `fetch-node-modules` (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 53dd2a1ae2

Tree-SHA512: aa4854b48d0186887a2db42b1f5e8fb27dc7da4ba6def50ee0a33d76ceb5425f593fb0c159e81809b67dda441559262872a331476136195c8033db94cc302026

flake.nix

@@ -73,7 +73,13 @@
         nbPkgs = self.lib.mkNbPkgs { inherit system pkgs; };
       in rec {
         packages = flake-utils.lib.flattenTree (removeAttrs nbPkgs [
-          "pinned" "modulesPkgs" "nixops19_09" "krops" "generate-secrets" "netns-exec"
+          "fetchNodeModules"
+          "krops"
+          "modulesPkgs"
+          "netns-exec"
+          "nixops19_09"
+          "pinned"
+          "generate-secrets"
         ]) // {
           inherit (import ./examples/qemu-vm/minimal-vm.nix self pkgs system)
             # A simple demo VM.

helper/update-fixed-output-derivation.sh

@@ -0,0 +1,24 @@
+set -euo pipefail
+
+# The file that defines the derivation that should be updated
+file=$1
+# The name of the output of this flake that should be updated
+flakeOutput=$2
+# A pattern in a line preceding the hash that should be updated
+patternPrecedingHash=$3
+
+sed -i "/$patternPrecedingHash/,/hash/ s|hash = .*|hash = \"\";|" $file
+# Display stderr and capture it. stdbuf is required to disable output buffering.
+stderr=$(
+    nix build --no-link -L .#$flakeOutput |&
+      stdbuf -oL grep -v '\berror:.*failed to build$' |
+      tee /dev/stderr || :
+)
+hash=$(echo "$stderr" | sed -nE 's/.*?\bgot: *?(sha256-.*)/\1/p')
+if [[ ! $hash ]]; then
+    echo
+    echo "Error: No hash in build output."
+    exit 1
+fi
+sed -i "/$patternPrecedingHash/,/hash/ s|hash = .*|hash = \"$hash\";|" $file
+echo "(Note: The above hash mismatch message is not an error. It is part of the fetching process.)"

pkgs/build-support/fetch-node-modules.nix

@@ -0,0 +1,74 @@
+# This is a modified version of
+# https://github.com/NixOS/nixpkgs/pull/128749
+
+{ lib, stdenvNoCC, makeWrapper, nodejs }:
+
+{ src
+, hash ? ""
+, runScripts ? false
+, preferLocalBuild ? true
+, npmFlags ? ""
+, ...
+} @ args:
+stdenvNoCC.mkDerivation ({
+  inherit src preferLocalBuild;
+
+  name = "${src.name}-node_modules";
+  nativeBuildInputs = [
+    makeWrapper
+    (if args ? nodejs then args.nodejs else nodejs)
+  ];
+
+  outputHashMode =  "recursive";
+
+  impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+
+  phases = "unpackPhase patchPhase buildPhase installPhase";
+
+  buildPhase = ''
+    runHook preBuild
+
+    if [[ ! -f package.json ]]; then
+        echo "Error: file `package.json` doesn't exist"
+        exit 1
+    fi
+    if [[ ! -f package-lock.json ]]; then
+        echo "Error: file `package-lock.json` doesn't exist"
+        exit 1
+    fi
+
+    export SOURCE_DATE_EPOCH=1
+    export npm_config_cache=/tmp
+    NPM_FLAGS="--omit=dev --omit=optional --no-update-notifier $npmFlags"
+    # Scripts may result in non-deterministic behavior.
+    # Some packages (e.g., Puppeteer) use postinstall scripts to download extra data.
+    if [[ ! $runScripts ]]; then
+        NPM_FLAGS+=" --ignore-scripts"
+    fi
+
+    echo "Running npm ci $NPM_FLAGS"
+    npm ci $NPM_FLAGS
+
+    cp package.json \
+       package-lock.json node_modules/
+    rm -f node_modules/.package-lock.json
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p $out/lib
+    cp -r node_modules $out/lib
+
+    runHook postInstall
+  '';
+} // (
+  if hash == "" then {
+    outputHashAlgo = "sha256";
+    outputHash = "";
+  } else {
+    outputHash = hash;
+  }
+) // (builtins.removeAttrs args [ "hash" ]))

pkgs/clightning-rest/composition.nix

@@ -1,17 +0,0 @@
-# This file has been generated by node2nix 1.9.0. Do not edit!
-
-{pkgs ? import <nixpkgs> {
-    inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}:
-
-let
-  nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
-    inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
-    inherit pkgs nodejs;
-    libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
-  };
-in
-import ./node-packages.nix {
-  inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit;
-  inherit nodeEnv;
-}

pkgs/clightning-rest/default.nix

@@ -1,16 +1,48 @@
-{ pkgs, lib, makeWrapper }:
-let
-  inherit (pkgs) nodejs;
-  nodePackages = import ./composition.nix { inherit pkgs nodejs; };
-in
-nodePackages.package.overrideAttrs (old: {
-  nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
+{ lib
+, stdenvNoCC
+, nodejs-16_x
+, nodejs-slim-16_x
+, fetchNodeModules
+, fetchurl
+, makeWrapper
+, rsync
+}:
+let self = stdenvNoCC.mkDerivation {
+  pname = "clightning-rest";
+  version = "0.8.0";
+
+  src = fetchurl {
+    url = "https://github.com/Ride-The-Lightning/c-lightning-REST/archive/refs/tags/v${self.version}.tar.gz";
+    hash = "sha256-Rg0/lN7exNFlsMj+HQcFwVqNRzCd1ztu56q5VIkglko=";
+  };
+
+  passthru = {
+    nodejs = nodejs-16_x;
+    nodejsRuntime = nodejs-slim-16_x;
+
+    nodeModules = fetchNodeModules {
+      inherit (self) src nodejs;
+      hash = "sha256-aG60RANqmWQ4sbm450MS2DWEoRksjj9/z6PoKBLtDB4=";
+    };
+  };
+
+  nativeBuildInputs = [
     makeWrapper
   ];
 
-  postInstall = ''
-    makeWrapper ${nodejs}/bin/node $out/bin/cl-rest \
-      --add-flags $out/lib/node_modules/c-lightning-rest/cl-rest
+  phases = "unpackPhase patchPhase installPhase";
+
+  installPhase = ''
+    dest=$out/lib/node_modules/clightning-rest
+    mkdir -p $dest
+    ${rsync}/bin/rsync -a --inplace * ${self.nodeModules}/lib/node_modules \
+      --exclude=/{screenshots,'*.Dockerfile'} \
+      $dest
+
+    makeWrapper ${self.nodejsRuntime}/bin/node $out/bin/cl-rest \
+      --add-flags $dest/cl-rest.js
+
+    runHook postInstall
   '';
 
   meta = with lib; {
@@ -20,4 +52,4 @@ nodePackages.package.overrideAttrs (old: {
     maintainers = with maintainers; [ nixbitcoin earvstedt ];
     platforms = platforms.unix;
   };
-})
+}; in self

pkgs/clightning-rest/generate.sh

@@ -1,44 +1,40 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i bash -p nodePackages.node2nix gnupg wget jq gnused
+#! nix-shell -i bash -p gnupg wget gnused
 set -euo pipefail
 
-TMPDIR="$(mktemp -d -p /tmp)"
-trap "rm -rf $TMPDIR" EXIT
-
-version="0.7.2"
+version="0.8.0"
 repo=https://github.com/Ride-The-Lightning/c-lightning-REST
 
+scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+
+updateSrc() {
+    TMPDIR="$(mktemp -d /tmp/clightning-rest.XXX)"
+    trap "rm -rf $TMPDIR" EXIT
+
     # Fetch and verify source tarball
-file=v${version}.tar.gz
-url=$repo/archive/refs/tags/$file
     export GNUPGHOME=$TMPDIR
+    # Fetch saubyk's key
     gpg --keyserver hkps://keyserver.ubuntu.com --recv-key 3E9BD4436C288039CA827A9200C9E2BC2E45666F
-wget -P $TMPDIR $url
+    file=v${version}.tar.gz
+    wget -P $TMPDIR $repo/archive/refs/tags/$file
     wget -P $TMPDIR $repo/releases/download/v${version}/$file.asc
     gpg --verify $TMPDIR/$file.asc $TMPDIR/$file
     hash=$(nix hash file $TMPDIR/$file)
 
-# Extract source
-src=$TMPDIR/src
-mkdir $src
-tar xvf $TMPDIR/$file -C $src --strip-components 1 >/dev/null
+    sed -i "
+      s|\bversion = .*;|version = \"$version\";|
+      s|\bhash = .*;|hash = \"$hash\";|
+    " default.nix
+}
 
-# Generate nix pkg
-node2nix \
-  --input $src/package.json \
-  --lock $src/package-lock.json \
-  --composition composition.nix \
-  --no-copy-node-env
+updateNodeModulesHash() {
+    $scriptDir/../../helper/update-fixed-output-derivation.sh ./default.nix clightning-rest.nodeModules nodeModules
+}
 
-# Use node-env.nix from nixpkgs
-nodeEnvImport='import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix"'
-sed -i "s|import ./node-env.nix|$nodeEnvImport|" composition.nix
-
-# Use the verified package src
-read -d '' fetchurl <<EOF || :
-fetchurl {
-      url = "$url";
-      hash = "$hash";
-    };
-EOF
-sed -i "s|src = .*/src;|src = ${fetchurl//$'\n'/\\n}|" node-packages.nix
+if [[ $# == 0 ]]; then
+    # Each of these can be run separately
+    updateSrc
+    updateNodeModulesHash
+else
+    eval "$@"
+fi

pkgs/clightning-rest/pkg.json

@@ -1,3 +0,0 @@
-[
-  { "c-lightning-REST": "file:./package" }
-]

pkgs/default.nix

@@ -10,13 +10,13 @@ in
   }
 }:
 let self = {
-  clightning-rest = pkgs.callPackage ./clightning-rest { };
+  clightning-rest = pkgs.callPackage ./clightning-rest { inherit (self) fetchNodeModules; };
   clboss = pkgs.callPackage ./clboss { };
   clightning-plugins = pkgs.recurseIntoAttrs (import ./clightning-plugins pkgs self.nbPython3Packages);
   joinmarket = pkgs.callPackage ./joinmarket { nbPythonPackageOverrides = import ./python-packages self; };
   lndinit = pkgs.callPackage ./lndinit { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
-  rtl = pkgs.callPackage ./rtl { };
+  rtl = pkgs.callPackage ./rtl { inherit (self) fetchNodeModules; };
   # The secp256k1 version used by joinmarket
   secp256k1 = pkgs.callPackage ./secp256k1 { };
   spark-wallet = pkgs.callPackage ./spark-wallet { };
@@ -25,6 +25,8 @@ let self = {
     packageOverrides = import ./python-packages self;
   }).pkgs;
 
+  fetchNodeModules = pkgs.callPackage ./build-support/fetch-node-modules.nix { };
+
   # Fix clightning build by using python package mistune 0.8.4, which is a
   # strict requirement. This version is affected by CVE-2022-34749, but this
   # is irrelevant in this context.

pkgs/rtl/composition.nix

@@ -1,17 +0,0 @@
-# This file has been generated by node2nix 1.9.0. Do not edit!
-
-{pkgs ? import <nixpkgs> {
-    inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}:
-
-let
-  nodeEnv = import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix" {
-    inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
-    inherit pkgs nodejs;
-    libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
-  };
-in
-import ./node-packages.nix {
-  inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit;
-  inherit nodeEnv;
-}

pkgs/rtl/default.nix

@@ -1,16 +1,67 @@
-{ pkgs, lib, makeWrapper }:
-let
-  nodejs = pkgs.nodejs-14_x;
-  nodePackages = import ./composition.nix { inherit pkgs nodejs; };
-in
-nodePackages.package.overrideAttrs (old: {
-  nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
+{ lib
+, stdenvNoCC
+, nodejs-16_x
+, nodejs-slim-16_x
+, fetchNodeModules
+, fetchpatch
+, fetchurl
+, applyPatches
+, makeWrapper
+}:
+let self = stdenvNoCC.mkDerivation {
+  pname = "rtl";
+  version = "0.13.0";
+
+  src = applyPatches {
+    src = fetchurl {
+      url = "https://github.com/Ride-The-Lightning/RTL/archive/refs/tags/v${self.version}.tar.gz";
+      hash = "sha256-6eopIO5ge6+vkNvZomjAB6e5Qi2kkhXpnSOWQQm53R0=";
+    };
+
+    patches = [
+      # Move non-runtime deps to `devDependencies`
+      # https://github.com/Ride-The-Lightning/RTL/pull/1070
+      (fetchpatch {
+        url = "https://github.com/Ride-The-Lightning/RTL/pull/1070.patch";
+        sha256 = "sha256-esDkYI27SNzj2AhYHS9XqlW0r2mr+o0K4A6PUE2kbWU=";
+      })
+    ];
+  };
+
+  passthru = {
+    nodejs = nodejs-16_x;
+    nodejsRuntime = nodejs-slim-16_x;
+
+    nodeModules = fetchNodeModules {
+      inherit (self) src nodejs;
+      hash = "sha256-LmCrf+lIXz9i9NVALYk9IOBUoPbW8fj+fUsyVJgAANI=";
+    };
+  };
+
+  nativeBuildInputs = [
     makeWrapper
   ];
 
-  postInstall = ''
-    makeWrapper ${nodejs}/bin/node $out/bin/rtl \
-      --add-flags $out/lib/node_modules/rtl/rtl
+  phases = "unpackPhase patchPhase installPhase";
+
+  # `src` already contains the precompiled frontend and backend.
+  # Copy all files required for packaging, like in
+  # https://github.com/Ride-The-Lightning/RTL/blob/master/dockerfiles/Dockerfile
+  installPhase = ''
+    dest=$out/lib/node_modules/rtl
+    mkdir -p $dest
+    cp -r \
+      rtl.js \
+      package.json \
+      frontend \
+      backend \
+      ${self.nodeModules}/lib/node_modules \
+      $dest
+
+    makeWrapper ${self.nodejsRuntime}/bin/node $out/bin/rtl \
+      --add-flags $dest/rtl.js
+
+    runHook postInstall
   '';
 
   meta = with lib; {
@@ -20,4 +71,4 @@ nodePackages.package.overrideAttrs (old: {
     maintainers = with maintainers; [ nixbitcoin earvstedt ];
     platforms = platforms.unix;
   };
-})
+}; in self

pkgs/rtl/generate.sh

@@ -1,44 +1,40 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i bash -p nodePackages.node2nix gnupg wget jq gnused
+#! nix-shell -i bash -p gnupg wget gnused
 set -euo pipefail
 
-TMPDIR="$(mktemp -d -p /tmp)"
-trap "rm -rf $TMPDIR" EXIT
-
-version="0.12.3"
+version="0.13.0"
 repo=https://github.com/Ride-The-Lightning/RTL
 
+scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+
+updateSrc() {
+    TMPDIR="$(mktemp -d /tmp/rtl.XXX)"
+    trap "rm -rf $TMPDIR" EXIT
+
     # Fetch and verify source tarball
-file=v${version}.tar.gz
-url=$repo/archive/refs/tags/$file
     export GNUPGHOME=$TMPDIR
+    # Fetch saubyk's key
     gpg --keyserver hkps://keyserver.ubuntu.com --recv-key 3E9BD4436C288039CA827A9200C9E2BC2E45666F
-wget -P $TMPDIR $url
+    file=v${version}.tar.gz
+    wget -P $TMPDIR $repo/archive/refs/tags/$file
     wget -P $TMPDIR $repo/releases/download/v${version}/$file.asc
     gpg --verify $TMPDIR/$file.asc $TMPDIR/$file
     hash=$(nix hash file $TMPDIR/$file)
 
-# Extract source
-src=$TMPDIR/src
-mkdir $src
-tar xvf $TMPDIR/$file -C $src --strip-components 1 >/dev/null
+    sed -i "
+      s|\bversion = .*;|version = \"$version\";|
+      s|\bhash = .*;|hash = \"$hash\";|
+    " default.nix
+}
 
-# Generate nix pkg
-node2nix \
-  --input $src/package.json \
-  --lock $src/package-lock.json \
-  --composition composition.nix \
-  --no-copy-node-env
+updateNodeModulesHash() {
+    $scriptDir/../../helper/update-fixed-output-derivation.sh ./default.nix rtl.nodeModules nodeModules
+}
 
-# Use node-env.nix from nixpkgs
-nodeEnvImport='import "${toString pkgs.path}/pkgs/development/node-packages/node-env.nix"'
-sed -i "s|import ./node-env.nix|$nodeEnvImport|" composition.nix
-
-# Use the verified package src
-read -d '' fetchurl <<EOF || :
-fetchurl {
-      url = "$url";
-      hash = "$hash";
-    };
-EOF
-sed -i "s|src = .*/src;|src = ${fetchurl//$'\n'/\\n}|" node-packages.nix
+if [[ $# == 0 ]]; then
+    # Each of these can be run separately
+    updateSrc
+    updateNodeModulesHash
+else
+    eval "$@"
+fi