From c22cb5f3a7b1c9520a0a97819e258a7768e78001 Mon Sep 17 00:00:00 2001 From: m3ta-chiron Date: Fri, 4 Sep 2026 08:04:34 +0200 Subject: [PATCH] n8n: 2.36.8 -> 2.37.9, migrate to pnpm 11, harden update.sh n8n >= 2.37.0 demands pnpm >= 11.22.0 via engines.pnpm and serialises patchedDependencies as plain scalars in pnpm-lock.yaml (same lockfileVersion '9.0'), which pnpm 10 cannot install. The pinned nixpkgs only ships pnpm_11 11.17.0, so: - build with pnpm_11 + fetcherVersion 4 and relax the engines.pnpm gate in package.json (relaxPnpmEngine) for both fetchPnpmDeps and the main build; swap pnpm_10 -> pnpm_11 in nativeBuildInputs - drop the workaround once nixpkgs ships pnpm >= 11.22 update.sh: detect ERR_PNPM_UNSUPPORTED_ENGINE / ERR_PNPM_LOCKFILE_CONFIG_MISMATCH with actionable hints, sync tests/n8n-overlay-test.nix expectations (checks were stale at 2.32.6), guard against leftover fake-hash sentinels, and run from the repo root so the relative nix --expr paths resolve. Verified: n8n-overlay check green, full n8n 2.37.9 build green. --- overlays/mods/n8n.nix | 38 +++++++++++++-- overlays/mods/update.sh | 99 +++++++++++++++++++++++++++++++------- tests/n8n-overlay-test.nix | 6 +-- 3 files changed, 117 insertions(+), 26 deletions(-) diff --git a/overlays/mods/n8n.nix b/overlays/mods/n8n.nix index 7125f45..744048f 100644 --- a/overlays/mods/n8n.nix +++ b/overlays/mods/n8n.nix @@ -4,24 +4,52 @@ # regular npm package `@e965/xlsx@0.20.3`, which already carries an integrity # field. The old lockfile-integrity workaround is therefore obsolete and has # been removed. + # + # n8n >= 2.37.0 demands pnpm >= 11.22.0 via `engines.pnpm`, but the pinned + # nixpkgs only ships pnpm_10 (10.34.5) and pnpm_11 (11.17.0) — both too old. + # pnpm enforces that gate for ITSELF regardless of `engine-strict`, so both + # fetchPnpmDeps and the main build die with ERR_PNPM_UNSUPPORTED_ENGINE + # before any hash mismatch is even reached. Additionally n8n's lockfile, + # while still declaring lockfileVersion '9.0', now serialises + # patchedDependencies as plain `name@version: hash` scalars (no `path:` + # mapping), which pnpm 10 cannot parse (ERR_PNPM_LOCKFILE_CONFIG_MISMATCH). + # We therefore build with pnpm_11 + fetcherVersion 4 and relax the engine + # gate in package.json. Drop the sed once nixpkgs ships pnpm >= 11.22 and + # this overlay's nixpkgs pin has caught up. + relaxPnpmEngine = '' + sed -i -E 's/"pnpm": *"[^"]*"/"pnpm": "*"/' package.json + ''; in prev.n8n.overrideAttrs (finalAttrs: previousAttrs: { - version = "2.36.8"; + version = "2.37.9"; src = prev.fetchFromGitHub { owner = "n8n-io"; repo = "n8n"; tag = "n8n@${finalAttrs.version}"; - hash = "sha256-2id5kyrISnYdhM/LPkyjZS5WrjeWHis/hu2zAMKY4wA="; + hash = "sha256-DFoXCtn+rM69ISKVV2aet8LfGCOsN29yuyYOpz0f71U="; }; pnpmDeps = prev.fetchPnpmDeps { inherit (finalAttrs) pname version src; - pnpm = prev.pnpm_10; - fetcherVersion = 3; - hash = "sha256-KCVNWw2at/282bwqYi9VHC6fSF7foQkPHme3M+CfmiE="; + pnpm = prev.pnpm_11; + fetcherVersion = 4; + postPatch = relaxPnpmEngine; + hash = "sha256-4SFl0JD4iQc4tQFtJvNWkYCVyT6sj7+id/BrJEaTl8E="; }; + postPatch = (previousAttrs.postPatch or "") + relaxPnpmEngine; + + # Swap pnpm_10 -> pnpm_11 from the inherited nativeBuildInputs: upstream + # n8n calls pnpm directly during the build (install/build/prune) and pnpm 10 + # cannot parse the pnpm-11 lockfile. The top-level pnpmConfigHook stays as + # is — it discovers the fetcher version from pnpmDeps' .fetcher-version. + nativeBuildInputs = builtins.map (x: + if (x.outPath or null) == prev.pnpm_10.outPath + then prev.pnpm_11 + else x) + previousAttrs.nativeBuildInputs; + preBuild = (previousAttrs.preBuild or "") + '' diff --git a/overlays/mods/update.sh b/overlays/mods/update.sh index f503f1d..41df3be 100755 --- a/overlays/mods/update.sh +++ b/overlays/mods/update.sh @@ -17,25 +17,52 @@ set -euo pipefail # 3. Compare to the version pinned in overlays/mods/n8n.nix # 4. Compute new src hash (fetchFromGitHub tarball) via nix-prefetch-url # 5. Compute new pnpmDeps hash via the fake-hash trick -# 6. Commit +# 6. Sync tests/n8n-overlay-test.nix expectations (version + both hashes) +# 7. Verify evaluation + test expectations, then commit SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NIX_FILE="$SCRIPT_DIR/n8n.nix" +TEST_FILE="$SCRIPT_DIR/../../tests/n8n-overlay-test.nix" NIXPKGS_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || true) -# Snapshot the current n8n.nix so we can restore it if anything goes wrong -# (error, SIGINT, SIGTERM, or a killed build) — otherwise a failed/aborted run -# leaves a stale version bump and/or the fake-hash sentinel behind. Restoring -# from the snapshot (rather than `git checkout`) also preserves any -# pre-existing working-tree edits the user may have made. The trap is disarmed -# once we reach a successful commit. +# A literal SRI sentinel (equal to lib.fakeHash). We write the literal rather +# than `lib.fakeHash` so we don't depend on `lib` being in scope inside +# n8n.nix — the overlay is imported with just `{ prev = pkgs; }`, so `lib` +# is not bound there. +FAKE_HASH='sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' + +if [[ -z "$NIXPKGS_ROOT" ]]; then + echo "ERROR: could not locate the git checkout root from $SCRIPT_DIR." >&2 + echo " The updater needs the flake context (flake.lock nixpkgs pin)." >&2 + exit 1 +fi + +# The `nix build/eval --expr` calls below resolve `./` against the current +# working directory, not against this script — always run from the repo root. +cd "$NIXPKGS_ROOT" + +# Snapshot the current n8n.nix (and the test expectations) so we can restore +# them if anything goes wrong (error, SIGINT, SIGTERM, or a killed build) — +# otherwise a failed/aborted run leaves a stale version bump and/or the +# fake-hash sentinel behind. Restoring from the snapshots (rather than +# `git checkout`) also preserves any pre-existing working-tree edits the user +# may have made. The trap is disarmed once we reach a successful commit. RESTORE_FILE="$(mktemp)" +TEST_RESTORE_FILE="$(mktemp)" cp "$NIX_FILE" "$RESTORE_FILE" +if [[ -f "$TEST_FILE" ]]; then + cp "$TEST_FILE" "$TEST_RESTORE_FILE" +else + : >"$TEST_RESTORE_FILE" +fi restore_on_failure() { if [[ -z "${UPDATE_SUCCEEDED:-}" ]]; then cp -f "$RESTORE_FILE" "$NIX_FILE" 2>/dev/null || true + if [[ -s "$TEST_RESTORE_FILE" ]]; then + cp -f "$TEST_RESTORE_FILE" "$TEST_FILE" 2>/dev/null || true + fi fi - rm -f "$RESTORE_FILE" + rm -f "$RESTORE_FILE" "$TEST_RESTORE_FILE" } trap restore_on_failure EXIT @@ -66,6 +93,13 @@ CURRENT=$(awk ' ' "$NIX_FILE") echo "Current version: $CURRENT" +if grep -q "$FAKE_HASH" "$NIX_FILE"; then + echo "ERROR: $NIX_FILE already contains the fake-hash sentinel." >&2 + echo " A previous update run was interrupted before it could restore." >&2 + echo " Fix up the file first (git diff / git checkout -- overlays/mods/n8n.nix)." >&2 + exit 1 +fi + if [[ "$VERSION" == "$CURRENT" ]]; then echo "Already at latest version, nothing to do." exit 0 @@ -97,12 +131,6 @@ awk -v sri="$SRC_SRI" ' # ── 5. Compute pnpmDeps hash via the fake-hash trick ─────────────────────── echo "==> Computing pnpmDeps hash (this may take a while) ..." -# A literal SRI sentinel (equal to lib.fakeHash). We write the literal rather -# than `lib.fakeHash` so we don't depend on `lib` being in scope inside -# n8n.nix — the overlay is imported with just `{ prev = pkgs; }`, so `lib` -# is not bound there. -FAKE_HASH='sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' - # 5a. Temporarily set the pnpmDeps hash to the fake sentinel. awk -v fake="\"$FAKE_HASH\"" ' !pnpm_started && /^[[:space:]]*pnpmDeps = / { pnpm_started = 1 } @@ -134,9 +162,19 @@ PNPM_HASH=$(echo "$BUILD_OUTPUT" | if [[ -z "$PNPM_HASH" ]]; then echo "ERROR: could not extract a pnpmDeps hash from the build output." >&2 - echo " This usually means fetchPnpmDeps failed for a reason OTHER than" >&2 - echo " a hash mismatch (e.g. a pnpm-lock patch in n8n.nix no longer applies," >&2 - echo " a tarball vanished, or the build was interrupted). Full output:" >&2 + if grep -q 'ERR_PNPM_UNSUPPORTED_ENGINE' <<<"$BUILD_OUTPUT"; then + echo " Cause: upstream n8n demands a newer pnpm than the pinned nixpkgs" >&2 + echo " ships (engines.pnpm gate). Check the relaxPnpmEngine workaround," >&2 + echo " the pnpm major and fetcherVersion in $NIX_FILE." >&2 + elif grep -q 'ERR_PNPM_LOCKFILE_CONFIG_MISMATCH' <<<"$BUILD_OUTPUT"; then + echo " Cause: the pinned pnpm cannot parse upstream's pnpm-lock.yaml" >&2 + echo " anymore (lockfile serialisation changed across pnpm majors)." >&2 + echo " Bump the pnpm major and fetcherVersion in $NIX_FILE." >&2 + else + echo " This usually means fetchPnpmDeps failed for a reason OTHER than" >&2 + echo " a hash mismatch (e.g. a pnpm-lock patch in n8n.nix no longer applies," >&2 + echo " a tarball vanished, or the build was interrupted). Full output:" >&2 + fi echo "$BUILD_OUTPUT" >&2 echo "Restoring n8n.nix via the EXIT trap." >&2 exit 1 @@ -149,6 +187,16 @@ echo " pnpmDeps hash: $PNPM_HASH" # plain literal substitution is sufficient. sed -i "s|$FAKE_HASH|$PNPM_HASH|" "$NIX_FILE" +# 5d. Keep the overlay test expectations (tests/n8n-overlay-test.nix) in sync — +# they pin version + both hashes and are evaluated by `nix flake check`. +if [[ -f "$TEST_FILE" ]]; then + sed -i \ + -e "s|expectedVersion = \"[^\"]*\"|expectedVersion = \"$VERSION\"|" \ + -e "s|expectedSrcHash = \"sha256-[A-Za-z0-9+/=]*\"|expectedSrcHash = \"$SRC_SRI\"|" \ + -e "s|expectedPnpmHash = \"sha256-[A-Za-z0-9+/=]*\"|expectedPnpmHash = \"$PNPM_HASH\"|" \ + "$TEST_FILE" +fi + # ── 6. Verify the file parses and commit ─────────────────────────────────── echo "==> Verifying nix evaluation ..." if ! nix eval --impure --expr " @@ -161,9 +209,24 @@ if ! nix eval --impure --expr " exit 1 fi +# The overlay test pins version + hashes and runs as part of `nix flake check`; +# make sure its expectations actually match the updated overlay before committing. +if [[ -f "$TEST_FILE" ]] && + ! nix eval --impure --no-warn-dirty --expr " + let + flake = builtins.getFlake (toString ./.); + check = flake.checks.\${builtins.currentSystem}.n8n-overlay or null; + in if check == null then null else check.drvPath +" >/dev/null 2>&1; then + echo "ERROR: tests/n8n-overlay-test.nix expectations do not match the updated" >&2 + echo " overlay — restoring files via the EXIT trap." >&2 + exit 1 +fi + if [[ -n "$NIXPKGS_ROOT" ]] && - [[ -n "$(git -C "$NIXPKGS_ROOT" status --porcelain "$NIX_FILE")" ]]; then + [[ -n "$(git -C "$NIXPKGS_ROOT" status --porcelain "$NIX_FILE" "$TEST_FILE")" ]]; then git -C "$NIXPKGS_ROOT" add "$NIX_FILE" + [[ -f "$TEST_FILE" ]] && git -C "$NIXPKGS_ROOT" add "$TEST_FILE" git -C "$NIXPKGS_ROOT" commit -m "n8n: $CURRENT -> $VERSION" echo "==> Committed" fi diff --git a/tests/n8n-overlay-test.nix b/tests/n8n-overlay-test.nix index 563a5e9..4dd384e 100644 --- a/tests/n8n-overlay-test.nix +++ b/tests/n8n-overlay-test.nix @@ -15,9 +15,9 @@ modifiedPkgs = mkPkgs [self.overlays.modifications]; defaultPkgs = mkPkgs [self.overlays.default]; - expectedVersion = "2.32.6"; - expectedSrcHash = "sha256-wWm6vGyJ2I2SBU38gRGaZG2FR5FBxH6V/sWQwn5B4Ac="; - expectedPnpmHash = "sha256-QzUJCF+VIku9/HrmiUR9FNCU3MlYtyKOILZjFNXvN8U="; + expectedVersion = "2.37.9"; + expectedSrcHash = "sha256-DFoXCtn+rM69ISKVV2aet8LfGCOsN29yuyYOpz0f71U="; + expectedPnpmHash = "sha256-4SFl0JD4iQc4tQFtJvNWkYCVyT6sj7+id/BrJEaTl8E="; expectedChangelog = "https://github.com/n8n-io/n8n/releases/tag/n8n@${expectedVersion}"; expectedRootNodeModuleSymlinks = [ "ln -s .pnpm/node_modules/sass-embedded node_modules/sass-embedded"