Upgrade matrix-sdk-crypto-nodejs and reverse-deps to fix build with new rust version (#348448)

nixos/modules/services/matrix/mjolnir.nix

@@ -186,6 +186,10 @@ in
       }
     ];
 
+    # This defaults to true in the application,
+    # which breaks older configs using pantalaimon or access tokens
+    services.mjolnir.settings.encryption.use = lib.mkDefault false;
+
     services.pantalaimon-headless.instances."mjolnir" = lib.mkIf cfg.pantalaimon.enable
       {
         homeserver = cfg.homeserverUrl;

pkgs/development/libraries/matrix-sdk-crypto-nodejs/beta3.nix

@@ -1,13 +1,76 @@
-import ./generic.nix {
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  cargo,
+  rustPlatform,
+  rustc,
+  napi-rs-cli,
+  nodejs,
+  libiconv,
+}:
+
+stdenv.mkDerivation rec {
+  pname = "matrix-sdk-crypto-nodejs";
   version = "0.1.0-beta.3";
-  hash = "sha256-0p+1cMn9PU+Jk2JW7G+sdzxhMaI3gEAk5w2nm05oBSU=";
-  outputHashes = {
-    "uniffi-0.21.0" = "sha256-blKCfCsSNtr8NtO7Let7VJ/9oGuW9Eu8j9A6/oHUcP0=";
+
+  src = fetchFromGitHub {
+    owner = "matrix-org";
+    repo = "matrix-rust-sdk";
+    rev = "${pname}-v${version}";
+    hash = "sha256-0p+1cMn9PU+Jk2JW7G+sdzxhMaI3gEAk5w2nm05oBSU=";
   };
-  cargoLock = ./Cargo-beta.3.lock;
+
   patches = [
     # This is needed because two versions of indexed_db_futures are present (which will fail to vendor, see https://github.com/rust-lang/cargo/issues/10310).
     # (matrix-sdk-crypto-nodejs doesn't use this dependency, we only need to remove it to vendor the dependencies successfully.)
     ./remove-duplicate-dependency.patch
   ];
+
+  cargoDeps = rustPlatform.importCargoLock {
+    lockFile = ./Cargo-beta.3.lock;
+    outputHashes = {
+      "uniffi-0.21.0" = "sha256-blKCfCsSNtr8NtO7Let7VJ/9oGuW9Eu8j9A6/oHUcP0=";
+    };
+  };
+
+  nativeBuildInputs = [
+    rustPlatform.cargoSetupHook
+    cargo
+    rustc
+    napi-rs-cli
+    nodejs
+  ];
+
+  buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
+
+  buildPhase = ''
+    runHook preBuild
+
+    cd bindings/${pname}
+    npm run release-build --offline
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    local -r outPath="$out/lib/node_modules/@matrix-org/${pname}"
+    mkdir -p "$outPath"
+    cp package.json index.js index.d.ts matrix-sdk-crypto.*.node "$outPath"
+
+    runHook postInstall
+  '';
+
+  meta = with lib; {
+    description = "No-network-IO implementation of a state machine that handles E2EE for Matrix clients";
+    homepage = "https://github.com/matrix-org/matrix-rust-sdk/tree/${src.rev}/bindings/matrix-sdk-crypto-nodejs";
+    license = licenses.asl20;
+    maintainers = with maintainers; [
+      winter
+      dandellion
+    ];
+    inherit (nodejs.meta) platforms;
+  };
 }

pkgs/development/libraries/matrix-sdk-crypto-nodejs/default.nix

@@ -1,11 +1,72 @@
-import ./generic.nix {
-  version = "0.1.0-beta.6";
-  hash = "sha256-0oLk7yq/XELS0GkeZj7PxY3KKXfzws0djF3KmxYisY0=";
-  outputHashes = {
-    "ruma-0.8.2" = "sha256-bKvcElIVugj+gZZhPFPGfCqva4fo1IqW/e9gf+q/Tfw=";
-    "uniffi-0.23.0" = "sha256-4WUp3PQm3ZgqHNMvz9+PBtNAeiq6m4PBViwXpQDglLk=";
-    "vodozemac-0.3.0" = "sha256-tAimsVD8SZmlVybb7HvRffwlNsfb7gLWGCplmwbLIVE=";
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  cargo,
+  rustPlatform,
+  rustc,
+  napi-rs-cli,
+  nodejs,
+  libiconv,
+}:
+
+stdenv.mkDerivation rec {
+  pname = "matrix-sdk-crypto-nodejs";
+  version = "0.2.0-beta.1";
+
+  src = fetchFromGitHub {
+    owner = "matrix-org";
+    repo = "matrix-rust-sdk-crypto-nodejs";
+    rev = "v${version}";
+    hash = "sha256-g86RPfhF9XHpbXhHRbyhl920VazCrQyRQrYV6tVCHy4=";
+  };
+
+  cargoDeps = rustPlatform.importCargoLock {
+    lockFile = ./Cargo.lock;
+    outputHashes = {
+      "matrix-sdk-base-0.7.0" = "sha256-nCiG4T/MB7gvGrmadKOEbh8+54081PHee9Bm8oY/nl0=";
+      "ruma-0.10.1" = "sha256-Yc5RKk4aRjNIoQsMl30fFehTDCkRO9VvenAvLoVHzXo=";
+      "vodozemac-0.6.0" = "sha256-jJgrJJ0SFcy2oRRZ3ubuKnM2pLO8Tx6NyXordWJjz8o=";
+    };
+  };
+
+  nativeBuildInputs = [
+    rustPlatform.cargoSetupHook
+    cargo
+    rustc
+    napi-rs-cli
+    nodejs
+  ];
+
+  buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
+
+  buildPhase = ''
+    runHook preBuild
+
+    npm run release-build --offline
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    local -r outPath="$out/lib/node_modules/@matrix-org/${pname}"
+    mkdir -p "$outPath"
+    cp package.json index.js index.d.ts matrix-sdk-crypto.*.node "$outPath"
+
+    runHook postInstall
+  '';
+
+  meta = with lib; {
+    description = "No-network-IO implementation of a state machine that handles E2EE for Matrix clients";
+    homepage = "https://github.com/matrix-org/matrix-rust-sdk-crypto-nodejs";
+    changelog = "https://github.com/matrix-org/matrix-rust-sdk-crypto-nodejs/blob/main/CHANGELOG.md";
+    license = licenses.asl20;
+    maintainers = with maintainers; [
+      winter
+      dandellion
+    ];
+    inherit (nodejs.meta) platforms;
   };
-  cargoLock = ./Cargo-beta.6.lock;
-  patches = [];
 }

pkgs/development/libraries/matrix-sdk-crypto-nodejs/generic.nix

@@ -1,60 +0,0 @@
-{ version, hash, outputHashes, cargoLock, patches }:
-
-{ lib, stdenv, fetchFromGitHub
-, cargo, rustPlatform, rustc, napi-rs-cli, nodejs, libiconv
-}:
-
-stdenv.mkDerivation rec {
-  pname = "matrix-sdk-crypto-nodejs";
-  inherit version patches;
-
-  src = fetchFromGitHub {
-    owner = "matrix-org";
-    repo = "matrix-rust-sdk";
-    rev = "${pname}-v${version}";
-    inherit hash;
-  };
-
-
-  cargoDeps = rustPlatform.importCargoLock {
-    lockFile = cargoLock;
-    inherit outputHashes;
-  };
-
-  nativeBuildInputs = [
-    rustPlatform.cargoSetupHook
-    cargo
-    rustc
-    napi-rs-cli
-    nodejs
-  ];
-
-  buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
-
-  buildPhase = ''
-    runHook preBuild
-
-    cd bindings/${pname}
-    npm run release-build --offline
-
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-
-    local -r outPath="$out/lib/node_modules/@matrix-org/${pname}"
-    mkdir -p "$outPath"
-    cp package.json index.js index.d.ts matrix-sdk-crypto.*.node "$outPath"
-
-    runHook postInstall
-  '';
-
-  meta = with lib; {
-    description = "No-network-IO implementation of a state machine that handles E2EE for Matrix clients";
-    homepage = "https://github.com/matrix-org/matrix-rust-sdk/tree/${src.rev}/bindings/matrix-sdk-crypto-nodejs";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ winter ];
-    inherit (nodejs.meta) platforms;
-  };
-}

pkgs/servers/mjolnir/001-disable-nsfwprotection.patch

@@ -0,0 +1,221 @@
+diff --git a/src/protections/NsfwProtection.ts b/src/protections/NsfwProtection.ts
+deleted file mode 100644
+index 8b6f8fd..0000000
+--- a/src/protections/NsfwProtection.ts
++++ /dev/null
+@@ -1,99 +0,0 @@
+-/*
+-Copyright 2024 The Matrix.org Foundation C.I.C.
+-
+-Licensed under the Apache License, Version 2.0 (the "License");
+-you may not use this file except in compliance with the License.
+-You may obtain a copy of the License at
+-
+-    http://www.apache.org/licenses/LICENSE-2.0
+-
+-Unless required by applicable law or agreed to in writing, software
+-distributed under the License is distributed on an "AS IS" BASIS,
+-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-See the License for the specific language governing permissions and
+-limitations under the License.
+-*/
+-
+-import { Protection } from "./IProtection";
+-import { Mjolnir } from "../Mjolnir";
+-import * as nsfw from 'nsfwjs';
+-import {LogLevel} from "@vector-im/matrix-bot-sdk";
+-import { node } from '@tensorflow/tfjs-node';
+-
+-
+-export class NsfwProtection extends Protection {
+-    settings = {};
+-    // @ts-ignore
+-    private model: any;
+-
+-    constructor() {
+-        super();
+-    }
+-
+-    async initialize() {
+-        this.model = await nsfw.load();
+-    }
+-
+-    public get name(): string {
+-        return 'NsfwProtection';
+-    }
+-
+-    public get description(): string {
+-        return "Scans all images sent into a protected room to determine if the image is " +
+-            "NSFW. If it is, the image will automatically be redacted.";
+-    }
+-
+-    public async handleEvent(mjolnir: Mjolnir, roomId: string, event: any): Promise<any> {
+-        if (event['type'] === 'm.room.message') {
+-            let content = JSON.stringify(event['content']);
+-            if (!content.toLowerCase().includes("mxc")) {
+-                return;
+-             }
+-            // try and grab a human-readable alias for more helpful management room output
+-            const maybeAlias = await mjolnir.client.getPublishedAlias(roomId)
+-            const room = maybeAlias ? maybeAlias : roomId
+-
+-            const mxcs = content.match(/(mxc?:\/\/[^\s'"]+)/gim);
+-            if (!mxcs) {
+-                //something's gone wrong with the regex
+-                await mjolnir.managementRoomOutput.logMessage(LogLevel.ERROR, "NSFWProtection", `Unable to find any mxcs in  ${event["event_id"]} in ${room}`);
+-                return;
+-            }
+-
+-            // @ts-ignore - see null check immediately above
+-            for (const mxc of mxcs) {
+-                const image = await mjolnir.client.downloadContent(mxc);
+-                const decodedImage = await node.decodeImage(image.data, 3);
+-                const predictions = await this.model.classify(decodedImage);
+-
+-
+-                for (const prediction of predictions) {
+-                    if (["Hentai", "Porn"].includes(prediction["className"])) {
+-                        if (prediction["probability"] > mjolnir.config.nsfwSensitivity) {
+-                            try {
+-                                await mjolnir.client.redactEvent(roomId, event["event_id"]);
+-                            } catch (err) {
+-                                await mjolnir.managementRoomOutput.logMessage(LogLevel.ERROR, "NSFWProtection", `There was an error redacting ${event["event_id"]} in ${room}: ${err}`);
+-                            }
+-                            let eventId = event["event_id"]
+-                            let body = `Redacted an image in ${room} ${eventId}`
+-                            let formatted_body = `<details>
+-                                                  <summary>Redacted an image in ${room}</summary>
+-                                                  <pre>${eventId}</pre>  <pre></pre>${room}</pre>
+-                                                  </details>`
+-                            const msg = {
+-                                msgtype: "m.notice",
+-                                body: body,
+-                                format: "org.matrix.custom.html",
+-                                formatted_body: formatted_body
+-                            };
+-                            await mjolnir.client.sendMessage(mjolnir.managementRoomId, msg);
+-                            break
+-                        }
+-                    }
+-                }
+-                decodedImage.dispose();
+-            }
+-        }
+-    }
+-}
+\ No newline at end of file
+diff --git a/src/protections/ProtectionManager.ts b/src/protections/ProtectionManager.ts
+index 9b84318..67f10dc 100644
+--- a/src/protections/ProtectionManager.ts
++++ b/src/protections/ProtectionManager.ts
+@@ -31,7 +31,6 @@ import { htmlEscape } from "../utils";
+ import { ERROR_KIND_FATAL, ERROR_KIND_PERMISSION } from "../ErrorCache";
+ import { RoomUpdateError } from "../models/RoomUpdateError";
+ import { LocalAbuseReports } from "./LocalAbuseReports";
+-import {NsfwProtection} from "./NsfwProtection";
+ import { MentionSpam } from "./MentionSpam";
+ 
+ const PROTECTIONS: Protection[] = [
+@@ -44,7 +43,6 @@ const PROTECTIONS: Protection[] = [
+     new DetectFederationLag(),
+     new JoinWaveShortCircuit(),
+     new LocalAbuseReports(),
+-    new NsfwProtection(),
+     new MentionSpam()
+ ];
+ 
+@@ -104,9 +102,6 @@ export class ProtectionManager {
+             protection.settings[key].setValue(value);
+         }
+         if (protection.enabled) {
+-            if (protection.name === "NsfwProtection") {
+-                (protection as NsfwProtection).initialize();
+-            }
+             for (let roomId of this.mjolnir.protectedRoomsTracker.getProtectedRooms()) {
+                 await protection.startProtectingRoom(this.mjolnir, roomId);
+             }
+diff --git a/test/integration/nsfwProtectionTest.ts b/test/integration/nsfwProtectionTest.ts
+deleted file mode 100644
+index c86fd38..0000000
+--- a/test/integration/nsfwProtectionTest.ts
++++ /dev/null
+@@ -1,78 +0,0 @@
+-import {newTestUser} from "./clientHelper";
+-
+-import {MatrixClient} from "@vector-im/matrix-bot-sdk";
+-import {getFirstReaction} from "./commands/commandUtils";
+-import {strict as assert} from "assert";
+-import { readFileSync } from 'fs';
+-
+-describe("Test: NSFW protection", function () {
+-    let client: MatrixClient;
+-    let room: string;
+-    this.beforeEach(async function () {
+-        client = await newTestUser(this.config.homeserverUrl, {name: {contains: "nsfw-protection"}});
+-        await client.start();
+-        const mjolnirId = await this.mjolnir.client.getUserId();
+-        room = await client.createRoom({ invite: [mjolnirId] });
+-        await client.joinRoom(room);
+-        await client.joinRoom(this.config.managementRoom);
+-        await client.setUserPowerLevel(mjolnirId, room, 100);
+-    })
+-    this.afterEach(async function () {
+-        await client.stop();
+-    })
+-
+-    function delay(ms: number) {
+-        return new Promise(resolve => setTimeout(resolve, ms));
+-    }
+-
+-
+-    it("Nsfw protection doesn't redact sfw images", async function() {
+-        this.timeout(20000);
+-
+-        await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir rooms add ${room}` });
+-        await getFirstReaction(client, this.mjolnir.managementRoomId, '✅', async () => {
+-                return await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir enable NsfwProtection` });
+-        });
+-
+-        const data = readFileSync('test_tree.jpg');
+-        const mxc = await client.uploadContent(data, 'image/png');
+-        let content = {"msgtype": "m.image", "body": "test.jpeg", "url": mxc};
+-        let imageMessage = await client.sendMessage(room, content);
+-
+-        await delay(500);
+-        let processedImage = await client.getEvent(room, imageMessage);
+-        assert.equal(Object.keys(processedImage.content).length, 3, "This event should not have been redacted");
+-    });
+-
+-    it("Nsfw protection redacts nsfw images", async function() {
+-        this.timeout(20000);
+-        // dial the sensitivity on the protection way up so that all images are flagged as NSFW
+-        this.mjolnir.config.nsfwSensitivity = 0.0;
+-
+-        await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir rooms add ${room}` });
+-        await getFirstReaction(client, this.mjolnir.managementRoomId, '✅', async () => {
+-                return await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir enable NsfwProtection` });
+-        });
+-
+-        const data = readFileSync('test_tree.jpg');
+-        const mxc = await client.uploadContent(data, 'image/png');
+-        let content = {"msgtype": "m.image", "body": "test.jpeg", "url": mxc};
+-        let imageMessage = await client.sendMessage(room, content);
+-
+-        let formatted_body = `<img src=${mxc} />`
+-        let htmlContent = {
+-                                msgtype: "m.image",
+-                                body: formatted_body,
+-                                format: "org.matrix.custom.html",
+-                                formatted_body: formatted_body
+-                            };
+-        let htmlMessage = await client.sendMessage(room, htmlContent)
+-
+-        await delay(500);
+-        let processedImage = await client.getEvent(room, imageMessage);
+-        assert.equal(Object.keys(processedImage.content).length, 0, "This event should have been redacted");
+-
+-        let processedHtml = await client.getEvent(room, htmlMessage)
+-        assert.equal(Object.keys(processedHtml.content).length, 0, "This html image event should have been redacted")
+-    });
+-});
+\ No newline at end of file

pkgs/servers/mjolnir/default.nix

@@ -10,20 +10,25 @@
 
 mkYarnPackage rec {
   pname = "mjolnir";
-  version = "1.6.5";
+  version = "1.8.3";
 
   src = fetchFromGitHub {
     owner = "matrix-org";
     repo = "mjolnir";
     rev = "refs/tags/v${version}";
-    hash = "sha256-xejFKz2MmdjMFU0X0SdI+qXTBRAwIvkcfZPQqXB9LV0=";
+    hash = "sha256-yD7QGsS2Em8Z95po9pGRUDmHgHe4z0j0Jnvy3IG7xKY=";
   };
 
+  patches = [
+    # TODO: Fix tfjs-node dependency
+    ./001-disable-nsfwprotection.patch
+  ];
+
   packageJSON = ./package.json;
 
   offlineCache = fetchYarnDeps {
     yarnLock = src + "/yarn.lock";
-    hash = "sha256-RpvdyxJj92k4wFjBBmWCnEpFVOXVWlHEm0SmEBUlnTM=";
+    hash = "sha256-05DqddK8+136Qq/JGeiITZkVJ8Dw9K9HfACKW86989U=";
   };
 
   packageResolutions = {

pkgs/servers/mjolnir/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mjolnir",
-  "version": "1.6.5",
+  "version": "1.8.3",
   "description": "A moderation tool for Matrix",
   "main": "lib/index.js",
   "repository": "git@github.com:matrix-org/mjolnir.git",
@@ -34,7 +34,7 @@
     "@types/pg": "^8.6.5",
     "@types/request": "^2.48.8",
     "@types/shell-quote": "1.7.1",
-    "crypto-js": "^4.1.1",
+    "crypto-js": "^4.2.0",
     "eslint": "^7.32",
     "expect": "^27.0.6",
     "mocha": "^9.0.1",
@@ -46,16 +46,20 @@
   "dependencies": {
     "@sentry/node": "^7.17.2",
     "@sentry/tracing": "^7.17.2",
+    "@tensorflow/tfjs-node": "^4.21.0",
+    "@vector-im/matrix-bot-sdk": "^0.7.1-element.6",
     "await-lock": "^2.2.2",
+    "axios": "^1.7.6",
     "body-parser": "^1.20.1",
     "config": "^3.3.8",
-    "express": "^4.17",
+    "express": "^4.20",
     "html-to-text": "^8.0.0",
     "humanize-duration": "^3.27.1",
     "humanize-duration-ts": "^2.1.1",
     "js-yaml": "^4.1.0",
     "jsdom": "^16.6.0",
-    "matrix-appservice-bridge": "8.1.2",
+    "matrix-appservice-bridge": "10.3.1",
+    "nsfwjs": "^4.1.0",
     "parse-duration": "^1.0.2",
     "pg": "^8.8.0",
     "prom-client": "^14.1.0",
@@ -64,6 +68,6 @@
     "yaml": "^2.2.2"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   }
 }

pkgs/top-level/all-packages.nix

@@ -9193,9 +9193,7 @@ with pkgs;
     nodejs = nodejs_18;
   };
 
-  matrix-appservice-discord = callPackage ../servers/matrix-appservice-discord {
-    matrix-sdk-crypto-nodejs = matrix-sdk-crypto-nodejs-0_1_0-beta_3;
-  };
+  matrix-appservice-discord = callPackage ../servers/matrix-appservice-discord { };
 
   matrix-corporal = callPackage ../servers/matrix-corporal { };
 
@@ -9310,9 +9308,7 @@ with pkgs;
 
   mitm6 = callPackage ../tools/security/mitm6 { };
 
-  mjolnir = callPackage ../servers/mjolnir {
-    matrix-sdk-crypto-nodejs = matrix-sdk-crypto-nodejs-0_1_0-beta_3;
-  };
+  mjolnir = callPackage ../servers/mjolnir { };
 
   mmutils = callPackage ../tools/X11/mmutils { };