nixos/rspamd-trainer: init; rspamd-trainer: init at unstable-2023-11-27

nixos/doc/manual/release-notes/rl-2405.section.md

@@ -26,6 +26,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [GNS3](https://www.gns3.com/), a network software emulator. Available as [services.gns3-server](#opt-services.gns3-server.enable).
 
+- [rspamd-trainer](https://gitlab.com/onlime/rspamd-trainer), script triggered by a helper which reads mails from a specific mail inbox and feeds them into rspamd for spam/ham training.
+
 - [Anki Sync Server](https://docs.ankiweb.net/sync-server.html), the official sync server built into recent versions of Anki. Available as [services.anki-sync-server](#opt-services.anki-sync-server.enable).
 The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been marked deprecated and will be dropped after 24.05 due to lack of maintenance of the anki-sync-server softwares.
 

nixos/modules/module-list.nix

@@ -620,6 +620,7 @@
   ./services/mail/public-inbox.nix
   ./services/mail/roundcube.nix
   ./services/mail/rspamd.nix
+  ./services/mail/rspamd-trainer.nix
   ./services/mail/rss2email.nix
   ./services/mail/schleuder.nix
   ./services/mail/spamassassin.nix

nixos/modules/services/mail/rspamd-trainer.nix

@@ -0,0 +1,76 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.services.rspamd-trainer;
+  format = pkgs.formats.toml { };
+
+in {
+  options.services.rspamd-trainer = {
+
+    enable = mkEnableOption (mdDoc "Spam/ham trainer for rspamd");
+
+    settings = mkOption {
+      default = { };
+      description = mdDoc ''
+        IMAP authentication configuration for rspamd-trainer. For supplying
+        the IMAP password, use the `secrets` option.
+      '';
+      type = types.submodule {
+        freeformType = format.type;
+      };
+      example = literalExpression ''
+        {
+          HOST = "localhost";
+          USERNAME = "spam@example.com";
+          INBOXPREFIX = "INBOX/";
+        }
+      '';
+    };
+
+    secrets = lib.mkOption {
+      type = with types; listOf path;
+      description = lib.mdDoc ''
+        A list of files containing the various secrets. Should be in the
+        format expected by systemd's `EnvironmentFile` directory. For the
+        IMAP account password use `PASSWORD = mypassword`.
+      '';
+      default = [ ];
+    };
+
+  };
+
+  config = mkIf cfg.enable {
+
+    systemd = {
+      services.rspamd-trainer = {
+        description = "Spam/ham trainer for rspamd";
+        serviceConfig = {
+          ExecStart = "${pkgs.rspamd-trainer}/bin/rspamd-trainer";
+          WorkingDirectory = "/var/lib/rspamd-trainer";
+          StateDirectory = [ "rspamd-trainer/log" ];
+          Type = "oneshot";
+          DynamicUser = true;
+          EnvironmentFile = [
+            ( format.generate "rspamd-trainer-env" cfg.settings )
+            cfg.secrets
+          ];
+        };
+      };
+      timers."rspamd-trainer" = {
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnBootSec = "10m";
+          OnUnitActiveSec = "10m";
+          Unit = "rspamd-trainer.service";
+        };
+      };
+    };
+
+  };
+
+  meta.maintainers = with lib.maintainers; [ onny ];
+
+}

nixos/tests/all-tests.nix

@@ -741,6 +741,7 @@ in {
   rosenpass = handleTest ./rosenpass.nix {};
   rshim = handleTest ./rshim.nix {};
   rspamd = handleTest ./rspamd.nix {};
+  rspamd-trainer = handleTest ./rspamd-trainer.nix {};
   rss2email = handleTest ./rss2email.nix {};
   rstudio-server = handleTest ./rstudio-server.nix {};
   rsyncd = handleTest ./rsyncd.nix {};

nixos/tests/rspamd-trainer.nix

@@ -0,0 +1,155 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+let
+  certs = import ./common/acme/server/snakeoil-certs.nix;
+  domain = certs.domain;
+in {
+  name = "rspamd-trainer";
+  meta = with pkgs.lib.maintainers; { maintainers = [ onny ]; };
+
+  nodes = {
+    machine = { options, config, ... }: {
+
+      security.pki.certificateFiles = [
+        certs.ca.cert
+      ];
+
+      networking.extraHosts = ''
+        127.0.0.1 ${domain}
+     '';
+
+      services.rspamd-trainer = {
+        enable = true;
+        settings = {
+          HOST = domain;
+          USERNAME = "spam@${domain}";
+          INBOXPREFIX = "INBOX/";
+        };
+        secrets = [
+          # Do not use this in production. This will make passwords
+          # world-readable in the Nix store
+          "${pkgs.writeText "secrets" ''
+            PASSWORD = test123
+          ''}"
+        ];
+      };
+
+      services.maddy = {
+        enable = true;
+        hostname = domain;
+        primaryDomain = domain;
+        ensureAccounts = [ "spam@${domain}" ];
+        ensureCredentials = {
+          # Do not use this in production. This will make passwords world-readable
+          # in the Nix store
+          "spam@${domain}".passwordFile = "${pkgs.writeText "postmaster" "test123"}";
+        };
+        tls = {
+          loader = "file";
+          certificates = [{
+            certPath = "${certs.${domain}.cert}";
+            keyPath = "${certs.${domain}.key}";
+          }];
+        };
+        config = builtins.replaceStrings [
+          "imap tcp://0.0.0.0:143"
+          "submission tcp://0.0.0.0:587"
+        ] [
+          "imap tls://0.0.0.0:993 tcp://0.0.0.0:143"
+          "submission tls://0.0.0.0:465 tcp://0.0.0.0:587"
+        ] options.services.maddy.config.default;
+      };
+
+      services.rspamd = {
+        enable = true;
+        locals = {
+          "redis.conf".text = ''
+            servers = "${config.services.redis.servers.rspamd.unixSocket}";
+          '';
+          "classifier-bayes.conf".text = ''
+            backend = "redis";
+            autolearn = true;
+          '';
+        };
+      };
+
+      services.redis.servers.rspamd = {
+        enable = true;
+        port = 0;
+        unixSocket = "/run/redis-rspamd/redis.sock";
+        user = config.services.rspamd.user;
+      };
+
+      environment.systemPackages = [
+        (pkgs.writers.writePython3Bin "send-testmail" { } ''
+          import smtplib
+          import ssl
+          from email.mime.text import MIMEText
+          context = ssl.create_default_context()
+          msg = MIMEText("Hello World")
+          msg['Subject'] = 'Test'
+          msg['From'] = "spam@${domain}"
+          msg['To'] = "spam@${domain}"
+          with smtplib.SMTP_SSL(host='${domain}', port=465, context=context) as smtp:
+              smtp.login('spam@${domain}', 'test123')
+              smtp.sendmail(
+                'spam@${domain}', 'spam@${domain}', msg.as_string()
+              )
+        '')
+        (pkgs.writers.writePython3Bin "create-mail-dirs" { } ''
+          import imaplib
+          with imaplib.IMAP4_SSL('${domain}') as imap:
+              imap.login('spam@${domain}', 'test123')
+              imap.create("\"INBOX/report_spam\"")
+              imap.create("\"INBOX/report_ham\"")
+              imap.create("\"INBOX/report_spam_reply\"")
+              imap.select("INBOX")
+              imap.copy("1", "\"INBOX/report_ham\"")
+              imap.logout()
+        '')
+        (pkgs.writers.writePython3Bin "test-imap" { } ''
+          import imaplib
+          with imaplib.IMAP4_SSL('${domain}') as imap:
+              imap.login('spam@${domain}', 'test123')
+              imap.select("INBOX/learned_ham")
+              status, refs = imap.search(None, 'ALL')
+              assert status == 'OK'
+              assert len(refs) == 1
+              status, msg = imap.fetch(refs[0], 'BODY[TEXT]')
+              assert status == 'OK'
+              assert msg[0][1].strip() == b"Hello World"
+              imap.logout()
+        '')
+      ];
+
+
+
+    };
+
+  };
+
+  testScript = { nodes }: ''
+    start_all()
+    machine.wait_for_unit("maddy.service")
+    machine.wait_for_open_port(143)
+    machine.wait_for_open_port(993)
+    machine.wait_for_open_port(587)
+    machine.wait_for_open_port(465)
+
+    # Send test mail to spam@domain
+    machine.succeed("send-testmail")
+
+    # Create mail directories required for rspamd-trainer and copy mail from
+    # INBOX into INBOX/report_ham
+    machine.succeed("create-mail-dirs")
+
+    # Start rspamd-trainer. It should read mail from INBOX/report_ham
+    machine.wait_for_unit("rspamd.service")
+    machine.wait_for_unit("redis-rspamd.service")
+    machine.wait_for_file("/run/rspamd/rspamd.sock")
+    machine.succeed("systemctl start rspamd-trainer.service")
+
+    # Check if mail got processed by rspamd-trainer successfully and check for
+    # it in INBOX/learned_ham
+    machine.succeed("test-imap")
+  '';
+})

pkgs/by-name/rs/rspamd-trainer/package.nix

@@ -0,0 +1,59 @@
+{ lib
+, python3
+, python3Packages
+, fetchFromGitLab
+, makeWrapper
+, stdenv
+, fetchpatch
+, rspamd
+}:
+
+python3Packages.buildPythonApplication {
+  pname = "rspamd-trainer";
+  version = "unstable-2023-11-27";
+  format = "pyproject";
+
+  src = fetchFromGitLab {
+    owner = "onlime";
+    repo = "rspamd-trainer";
+    rev = "eb6639a78a019ade6781f3a8418eddc030f8fa14";
+    hash = "sha256-Me6WZhQ6SvDGGBQQtSA/7bIfKtsz6D5rvQeU12sVzgY=";
+  };
+
+  patches = [
+    # Refactor pyproject.toml
+    # https://gitlab.com/onlime/rspamd-trainer/-/merge_requests/2
+    (fetchpatch {
+      url = "https://gitlab.com/onlime/rspamd-trainer/-/commit/8824bfb9a9826988a90a401b8e51c20f5366ed70.patch";
+      hash = "sha256-qiXfwMUfM/iV+fHba8xdwQD92RQz627+HdUTgwgRZdc=";
+      name = "refactor_pyproject.patch";
+    })
+  ];
+
+  postPatch = ''
+    # Fix module path not applied by patch
+    mv helper src/
+    touch src/helper/__init__.py
+    mv settings.py src/rspamd_trainer/
+    sed -i 's/from settings/from .settings/' src/rspamd_trainer/run.py
+
+    # Fix rspamc path
+    sed -i "s|/usr/bin/rspamc|${rspamd}/bin/rspamc|" src/rspamd_trainer/run.py
+  '';
+
+  nativeBuildInputs = with python3.pkgs; [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = with python3.pkgs; [
+    python-dotenv
+    imapclient
+  ];
+
+  meta = {
+    homepage = "https://gitlab.com/onlime/rspamd-trainer";
+    description = "Grabs messages from a spam mailbox via IMAP and feeds them to Rspamd for training";
+    license = lib.licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ onny ];
+  };
+}