From f15212aad86ad80fb355c8071c1e1e9a9ea879c7 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Thu, 6 Jul 2023 22:36:22 +0200
Subject: [PATCH 01/16] nixos/synapse: cleanup, split out listener type and
 service config

---
 nixos/modules/services/matrix/synapse.nix | 369 ++++++++++++----------
 1 file changed, 195 insertions(+), 174 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index 71f64d2fc4f8..1dc21167175e 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -4,12 +4,16 @@ with lib;
 
 let
   cfg = config.services.matrix-synapse;
-  format = pkgs.formats.yaml {};
+  format = pkgs.formats.yaml { };
 
   # remove null values from the final configuration
   finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings;
   configFile = format.generate "homeserver.yaml" finalSettings;
 
+  pluginsEnv = cfg.package.python.buildEnv.override {
+    extraLibs = cfg.plugins;
+  };
+
   usePostgresql = cfg.settings.database.name == "psycopg2";
   hasLocalPostgresDB = let args = cfg.settings.database.args; in
     usePostgresql && (!(args ? host) || (elem args.host [ "localhost" "127.0.0.1" "::1" ]));
@@ -154,7 +158,106 @@ in {
 
   ];
 
-  options = {
+  options = let
+    listenerType = types.submodule {
+      options = {
+        port = mkOption {
+          type = types.port;
+          example = 8448;
+          description = lib.mdDoc ''
+            The port to listen for HTTP(S) requests on.
+          '';
+        };
+
+        bind_addresses = mkOption {
+          type = types.listOf types.str;
+          default = [
+            "::1"
+            "127.0.0.1"
+          ];
+          example = literalExpression ''
+            [
+              "::"
+              "0.0.0.0"
+            ]
+          '';
+          description = lib.mdDoc ''
+            IP addresses to bind the listener to.
+          '';
+        };
+
+        type = mkOption {
+          type = types.enum [
+            "http"
+            "manhole"
+            "metrics"
+            "replication"
+          ];
+          default = "http";
+          example = "metrics";
+          description = lib.mdDoc ''
+            The type of the listener, usually http.
+          '';
+        };
+
+        tls = mkOption {
+          type = types.bool;
+          default = true;
+          example = false;
+          description = lib.mdDoc ''
+            Whether to enable TLS on the listener socket.
+          '';
+        };
+
+        x_forwarded = mkOption {
+          type = types.bool;
+          default = false;
+          example = true;
+          description = lib.mdDoc ''
+            Use the X-Forwarded-For (XFF) header as the client IP and not the
+            actual client IP.
+          '';
+        };
+
+        resources = mkOption {
+          type = types.listOf (types.submodule {
+            options = {
+              names = mkOption {
+                type = types.listOf (types.enum [
+                  "client"
+                  "consent"
+                  "federation"
+                  "keys"
+                  "media"
+                  "metrics"
+                  "openid"
+                  "replication"
+                  "static"
+                ]);
+                description = lib.mdDoc ''
+                  List of resources to host on this listener.
+                '';
+                example = [
+                  "client"
+                ];
+              };
+              compress = mkOption {
+                type = types.bool;
+                description = lib.mdDoc ''
+                  Should synapse compress HTTP responses to clients that support it?
+                  This should be disabled if running synapse behind a load balancer
+                  that can do automatic compression.
+                '';
+              };
+            };
+          });
+          description = lib.mdDoc ''
+            List of HTTP resources to serve on this listener.
+          '';
+        };
+      };
+    };
+  in {
     services.matrix-synapse = {
       enable = mkEnableOption (lib.mdDoc "matrix.org synapse");
 
@@ -251,7 +354,7 @@ in {
       };
 
       settings = mkOption {
-        default = {};
+        default = { };
         description = mdDoc ''
           The primary synapse configuration. See the
           [sample configuration](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_config.yaml)
@@ -409,118 +512,21 @@ in {
             };
 
             listeners = mkOption {
-              type = types.listOf (types.submodule {
-                options = {
-                  port = mkOption {
-                    type = types.port;
-                    example = 8448;
-                    description = lib.mdDoc ''
-                      The port to listen for HTTP(S) requests on.
-                    '';
-                  };
-
-                  bind_addresses = mkOption {
-                    type = types.listOf types.str;
-                    default = [
-                      "::1"
-                      "127.0.0.1"
-                    ];
-                    example = literalExpression ''
-                    [
-                      "::"
-                      "0.0.0.0"
-                    ]
-                    '';
-                    description = lib.mdDoc ''
-                     IP addresses to bind the listener to.
-                    '';
-                  };
-
-                  type = mkOption {
-                    type = types.enum [
-                      "http"
-                      "manhole"
-                      "metrics"
-                      "replication"
-                    ];
-                    default = "http";
-                    example = "metrics";
-                    description = lib.mdDoc ''
-                      The type of the listener, usually http.
-                    '';
-                  };
-
-                  tls = mkOption {
-                    type = types.bool;
-                    default = true;
-                    example = false;
-                    description = lib.mdDoc ''
-                      Whether to enable TLS on the listener socket.
-                    '';
-                  };
-
-                  x_forwarded = mkOption {
-                    type = types.bool;
-                    default = false;
-                    example = true;
-                    description = lib.mdDoc ''
-                      Use the X-Forwarded-For (XFF) header as the client IP and not the
-                      actual client IP.
-                    '';
-                  };
-
-                  resources = mkOption {
-                    type = types.listOf (types.submodule {
-                      options = {
-                        names = mkOption {
-                          type = types.listOf (types.enum [
-                            "client"
-                            "consent"
-                            "federation"
-                            "keys"
-                            "media"
-                            "metrics"
-                            "openid"
-                            "replication"
-                            "static"
-                          ]);
-                          description = lib.mdDoc ''
-                            List of resources to host on this listener.
-                          '';
-                          example = [
-                            "client"
-                          ];
-                        };
-                        compress = mkOption {
-                          type = types.bool;
-                          description = lib.mdDoc ''
-                            Should synapse compress HTTP responses to clients that support it?
-                            This should be disabled if running synapse behind a load balancer
-                            that can do automatic compression.
-                          '';
-                        };
-                      };
-                    });
-                    description = lib.mdDoc ''
-                      List of HTTP resources to serve on this listener.
-                    '';
-                  };
-                };
-              });
-              default = [ {
+              type = types.listOf listenerType;
+              default = [{
                 port = 8008;
                 bind_addresses = [ "127.0.0.1" ];
                 type = "http";
                 tls = false;
                 x_forwarded = true;
-                resources = [ {
+                resources = [{
                   names = [ "client" ];
                   compress = true;
                 } {
                   names = [ "federation" ];
                   compress = false;
-                } ];
-              } ];
+                }];
+              }];
               description = lib.mdDoc ''
                 List of ports that Synapse should listen on, their purpose and their configuration.
               '';
@@ -534,7 +540,7 @@ in {
               default = if versionAtLeast config.system.stateVersion "18.03"
                 then "psycopg2"
                 else "sqlite3";
-               defaultText = literalExpression ''
+              defaultText = literalExpression ''
                 if versionAtLeast config.system.stateVersion "18.03"
                 then "psycopg2"
                 else "sqlite3"
@@ -551,10 +557,10 @@ in {
                 psycopg2 = "matrix-synapse";
               }.${cfg.settings.database.name};
               defaultText = literalExpression ''
-              {
-                sqlite3 = "''${${options.services.matrix-synapse.dataDir}}/homeserver.db";
-                psycopg2 = "matrix-synapse";
-              }.''${${options.services.matrix-synapse.settings}.database.name};
+                {
+                  sqlite3 = "''${${options.services.matrix-synapse.dataDir}}/homeserver.db";
+                  psycopg2 = "matrix-synapse";
+                }.''${${options.services.matrix-synapse.settings}.database.name};
               '';
               description = lib.mdDoc ''
                 Name of the database when using the psycopg2 backend,
@@ -622,7 +628,7 @@ in {
 
             url_preview_ip_range_whitelist = mkOption {
               type = types.listOf types.str;
-              default = [];
+              default = [ ];
               description = lib.mdDoc ''
                 List of IP address CIDR ranges that the URL preview spider is allowed
                 to access even if they are specified in url_preview_ip_range_blacklist.
@@ -644,7 +650,7 @@ in {
                     on how to configure it properly.
                   ''))
                   (types.attrsOf types.str));
-              default = [];
+              default = [ ];
               example = literalExpression ''
                 [
                   { scheme = "http"; } # no http previews
@@ -690,7 +696,7 @@ in {
 
             turn_uris = mkOption {
               type = types.listOf types.str;
-              default = [];
+              default = [ ];
               example = [
                 "turn:turn.example.com:3487?transport=udp"
                 "turn:turn.example.com:3487?transport=tcp"
@@ -727,12 +733,12 @@ in {
                   };
                 };
               });
-              default = [ {
+              default = [{
                 server_name = "matrix.org";
                 verify_keys = {
                   "ed25519:auto" = "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw";
                 };
-              } ];
+              }];
               description = lib.mdDoc ''
                 The trusted servers to download signing keys from.
               '';
@@ -752,7 +758,7 @@ in {
 
       extraConfigFiles = mkOption {
         type = types.listOf types.path;
-        default = [];
+        default = [ ];
         description = lib.mdDoc ''
           Extra config files to include.
 
@@ -767,7 +773,8 @@ in {
 
   config = mkIf cfg.enable {
     assertions = [
-      { assertion = hasLocalPostgresDB -> config.services.postgresql.enable;
+      {
+        assertion = hasLocalPostgresDB -> config.services.postgresql.enable;
         message = ''
           Cannot deploy matrix-synapse with a configuration for a local postgresql database
             and a missing postgresql service. Since 20.03 it's mandatory to manually configure the
@@ -803,65 +810,79 @@ in {
       gid = config.ids.gids.matrix-synapse;
     };
 
-    systemd.services.matrix-synapse = {
-      description = "Synapse Matrix homeserver";
-      after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
-      wantedBy = [ "multi-user.target" ];
-      preStart = ''
-        ${cfg.package}/bin/synapse_homeserver \
-          --config-path ${configFile} \
-          --keys-directory ${cfg.dataDir} \
-          --generate-keys
-      '';
-      environment = optionalAttrs (cfg.withJemalloc) {
-        LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
-      };
-      serviceConfig = {
-        Type = "notify";
-        User = "matrix-synapse";
-        Group = "matrix-synapse";
-        WorkingDirectory = cfg.dataDir;
-        ExecStartPre = [ ("+" + (pkgs.writeShellScript "matrix-synapse-fix-permissions" ''
-          chown matrix-synapse:matrix-synapse ${cfg.settings.signing_key_path}
-          chmod 0600 ${cfg.settings.signing_key_path}
-        '')) ];
-        ExecStart = ''
-          ${cfg.package}/bin/synapse_homeserver \
-            ${ concatMapStringsSep "\n  " (x: "--config-path ${x} \\") ([ configFile ] ++ cfg.extraConfigFiles) }
-            --keys-directory ${cfg.dataDir}
-        '';
-        ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID";
-        Restart = "on-failure";
-        UMask = "0077";
+    systemd.services =
+      let
+        baseServiceConfig = {
+          after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
+          wantedBy = [ "multi-user.target" ];
+          environment = optionalAttrs (cfg.withJemalloc) {
+            LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
+          };
+          serviceConfig = {
+            Type = "notify";
+            User = "matrix-synapse";
+            Group = "matrix-synapse";
+            WorkingDirectory = cfg.dataDir;
+            ExecReload = "${pkgs.util-linux}/bin/kill -HUP $MAINPID";
+            Restart = "on-failure";
+            UMask = "0077";
 
-        # Security Hardening
-        # Refer to systemd.exec(5) for option descriptions.
-        CapabilityBoundingSet = [ "" ];
-        LockPersonality = true;
-        NoNewPrivileges = true;
-        PrivateDevices = true;
-        PrivateTmp = true;
-        PrivateUsers = true;
-        ProcSubset = "pid";
-        ProtectClock = true;
-        ProtectControlGroups = true;
-        ProtectHome = true;
-        ProtectHostname = true;
-        ProtectKernelLogs = true;
-        ProtectKernelModules = true;
-        ProtectKernelTunables = true;
-        ProtectProc = "invisible";
-        ProtectSystem = "strict";
-        ReadWritePaths = [ cfg.dataDir ];
-        RemoveIPC = true;
-        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
-        RestrictNamespaces = true;
-        RestrictRealtime = true;
-        RestrictSUIDSGID = true;
-        SystemCallArchitectures = "native";
-        SystemCallFilter = [ "@system-service" "~@resources" "~@privileged" ];
+            # Security Hardening
+            # Refer to systemd.exec(5) for option descriptions.
+            CapabilityBoundingSet = [ "" ];
+            LockPersonality = true;
+            NoNewPrivileges = true;
+            PrivateDevices = true;
+            PrivateTmp = true;
+            PrivateUsers = true;
+            ProcSubset = "pid";
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHome = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            ProtectProc = "invisible";
+            ProtectSystem = "strict";
+            ReadWritePaths = [ cfg.dataDir ];
+            RemoveIPC = true;
+            RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            RestrictSUIDSGID = true;
+            SystemCallArchitectures = "native";
+            SystemCallFilter = [ "@system-service" "~@resources" "~@privileged" ];
+          };
+        };
+      in
+      {
+        matrix-synapse = lib.mkMerge [
+          baseServiceConfig
+          {
+            description = "Synapse Matrix homeserver";
+            preStart = ''
+              ${cfg.package}/bin/synapse_homeserver \
+                --config-path ${configFile} \
+                --keys-directory ${cfg.dataDir} \
+                --generate-keys
+            '';
+            serviceConfig = {
+              ExecStartPre = [
+                ("+" + (pkgs.writeShellScript "matrix-synapse-fix-permissions" ''
+                  chown matrix-synapse:matrix-synapse ${cfg.settings.signing_key_path}
+                  chmod 0600 ${cfg.settings.signing_key_path}
+                ''))
+              ];
+              ExecStart = ''
+                ${cfg.package}/bin/synapse_homeserver \
+                  ${ concatMapStringsSep "\n  " (x: "--config-path ${x} \\") ([ configFile ] ++ cfg.extraConfigFiles) }
+                  --keys-directory ${cfg.dataDir}
+              '';
+            };
+          }
+        ];
       };
-    };
 
     environment.systemPackages = [ registerNewMatrixUser ];
   };

From b7c41da8d68c7e82c9957cc530e1141f0ca5aba6 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Thu, 6 Jul 2023 22:28:15 +0200
Subject: [PATCH 02/16] nixos/synapse: update listener settings

The resource type health is currently missing, but should be available
according to https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#listeners
---
 nixos/modules/services/matrix/synapse.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index 1dc21167175e..decf9c42c848 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -227,6 +227,7 @@ in {
                   "client"
                   "consent"
                   "federation"
+                  "health"
                   "keys"
                   "media"
                   "metrics"
@@ -242,9 +243,10 @@ in {
                 ];
               };
               compress = mkOption {
+                default = false;
                 type = types.bool;
                 description = lib.mdDoc ''
-                  Should synapse compress HTTP responses to clients that support it?
+                  Whether synapse should compress HTTP responses to clients that support it.
                   This should be disabled if running synapse behind a load balancer
                   that can do automatic compression.
                 '';

From b32918012814fcca1c5cd6c815b5565d71e96610 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Thu, 6 Jul 2023 23:03:51 +0200
Subject: [PATCH 03/16] nixos/synapse: add option to configure redis
 automatically

---
 nixos/modules/services/matrix/synapse.nix | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index decf9c42c848..60ad1fa42bc7 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -770,6 +770,14 @@ in {
           NixOps is in use.
         '';
       };
+
+      configureRedisLocally = lib.mkOption {
+        type = types.bool;
+        default = false;
+        description = lib.mdDoc ''
+          Whether to automatically configure a local redis server for matrix-synapse.
+        '';
+      };
     };
   };
 
@@ -794,6 +802,11 @@ in {
       }
     ];
 
+    services.matrix-synapse.settings.redis = lib.mkIf cfg.configureRedisLocally {
+      enabled = true;
+      path = config.services.redis.servers.matrix-synapse.unixSocket;
+    };
+
     services.matrix-synapse.configFile = configFile;
     services.matrix-synapse.package = wrapped;
 
@@ -886,6 +899,11 @@ in {
         ];
       };
 
+    services.redis.servers.matrix-synapse = lib.mkIf cfg.configureRedisLocally {
+      enable = true;
+      user = "matrix-synapse";
+    };
+
     environment.systemPackages = [ registerNewMatrixUser ];
   };
 

From 72a26e2b5465b967c462e08b8c64151356683c17 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Thu, 6 Jul 2023 22:42:36 +0200
Subject: [PATCH 04/16] nixos/synapse: add options to configure workers

---
 .../manual/release-notes/rl-2311.section.md   |  2 +
 nixos/modules/services/matrix/synapse.nix     | 99 +++++++++++++++++--
 2 files changed, 93 insertions(+), 8 deletions(-)

diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md
index 82dbe187d957..24aaf9d79ebb 100644
--- a/nixos/doc/manual/release-notes/rl-2311.section.md
+++ b/nixos/doc/manual/release-notes/rl-2311.section.md
@@ -242,6 +242,8 @@
 Unfortunately all servers supporting new clients (newer version of anki-sync-server, anki's built in sync server and this new rust package) do not support the older sync protocol that was used in the old server, so such old clients will also need updating and in particular the anki package in nixpkgs is also being updated in this release.
 The module update takes care of the new config syntax and the data itself (user login and cards) are compatible, so users of the module will be able to just log in again after updating both client and server without any extra action.
 
+- `services.matrix-synapse` has new options to configure worker processes for matrix-synapse using `services.matrix-synapse.workers.enable` and `services.matrix-synapse.workers.config`. It's also now possible to configure a local redis server using `services.matrix-synapse.configureRedisLocally`.
+
 - `services.nginx` gained a `defaultListen` option at server-level with support for PROXY protocol listeners, also `proxyProtocol` is now exposed in `services.nginx.virtualHosts.<name>.listen` option. It is now possible to run PROXY listeners and non-PROXY listeners at a server-level, see [#213510](https://github.com/NixOS/nixpkgs/pull/213510/) for more details.
 
 - `services.restic.backups` now adds wrapper scripts to your system path, which set the same environment variables as the service, so restic operations can easly be run from the command line. This behavior can be disabled by setting `createWrapper` to `false`, per backup configuration.
diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index 60ad1fa42bc7..b7591f984730 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -10,13 +10,10 @@ let
   finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings;
   configFile = format.generate "homeserver.yaml" finalSettings;
 
-  pluginsEnv = cfg.package.python.buildEnv.override {
-    extraLibs = cfg.plugins;
-  };
-
   usePostgresql = cfg.settings.database.name == "psycopg2";
   hasLocalPostgresDB = let args = cfg.settings.database.args; in
     usePostgresql && (!(args ? host) || (elem args.host [ "localhost" "127.0.0.1" "::1" ]));
+  hasWorkers = cfg.workers.enable && (cfg.workers.config != { });
 
   registerNewMatrixUser =
     let
@@ -758,6 +755,45 @@ in {
         };
       };
 
+      workers = lib.mkOption {
+        default = { };
+        description = lib.mdDoc ''
+          Options for configuring workers. See `services.matrix-synapse.workers.enable`
+          for a more detailed description.
+        '';
+        type = types.submodule {
+          options = {
+            enable = lib.mkOption {
+              type = types.bool;
+              default = false;
+              description = lib.mdDoc ''
+                Whether to enable matrix synapse workers
+              '';
+            };
+            config = lib.mkOption {
+              type = types.attrsOf (types.submodule {
+                freeformType = format.type;
+                options = {
+                  worker_listeners = lib.mkOption {
+                    default = [ ];
+                    type = types.listOf listenerType;
+                    description = lib.mdDoc ''
+                      List of ports that this worker should listen on, their purpose and their configuration.
+                    '';
+                  };
+                };
+              });
+              default = { };
+              description = lib.mdDoc ''
+                List of workers to configure. See the
+                [worker documention](https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration)
+                for possible values.
+              '';
+            };
+          };
+        };
+      };
+
       extraConfigFiles = mkOption {
         type = types.listOf types.path;
         default = [ ];
@@ -800,6 +836,13 @@ in {
           For further information about this update, please read the release-notes of 20.03 carefully.
         '';
       }
+      {
+        assertion = hasWorkers -> cfg.settings.redis.enabled;
+        message = ''
+          Workers for matrix-synapse require configuring a redis instance. This can be done
+          automatically by setting `services.matrix-synapse.configureRedisLocally = true`.
+        '';
+      }
     ];
 
     services.matrix-synapse.settings.redis = lib.mkIf cfg.configureRedisLocally {
@@ -825,11 +868,26 @@ in {
       gid = config.ids.gids.matrix-synapse;
     };
 
+    systemd.targets.matrix-synapse = lib.mkIf hasWorkers {
+      description = "Synapse Matrix parent target";
+      after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
+      wantedBy = [ "multi-user.target" ];
+    };
+
     systemd.services =
       let
+        targetConfig =
+          if hasWorkers
+          then {
+            partOf = [ "matrix-synapse.target" ];
+            wantedBy = [ "matrix-synapse.target" ];
+            unitConfig.ReloadPropagatedFrom = "matrix-synapse.target";
+          }
+          else {
+            after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
+            wantedBy = [ "multi-user.target" ];
+          };
         baseServiceConfig = {
-          after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service";
-          wantedBy = [ "multi-user.target" ];
           environment = optionalAttrs (cfg.withJemalloc) {
             LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so";
           };
@@ -869,7 +927,31 @@ in {
             SystemCallArchitectures = "native";
             SystemCallFilter = [ "@system-service" "~@resources" "~@privileged" ];
           };
-        };
+        }
+        // targetConfig;
+        genWorkerService = name: workerCfg:
+          let
+            finalWorkerCfg = workerCfg // { worker_name = name; };
+            workerConfigFile = format.generate "worker-${name}.yaml" finalWorkerCfg;
+          in
+          {
+            name = "matrix-synapse-worker-${name}";
+            value = lib.mkMerge [
+              baseServiceConfig
+              {
+                description = "Synapse Matrix worker ${name}";
+                # make sure the main process starts first for potential database migrations
+                after = [ "matrix-synapse.service" ];
+                serviceConfig = {
+                  ExecStart = ''
+                    ${cfg.package}/bin/synapse_worker \
+                      ${ concatMapStringsSep "\n  " (x: "--config-path ${x} \\") ([ configFile workerConfigFile ] ++ cfg.extraConfigFiles) }
+                      --keys-directory ${cfg.dataDir}
+                  '';
+                };
+              }
+            ];
+          };
       in
       {
         matrix-synapse = lib.mkMerge [
@@ -897,7 +979,8 @@ in {
             };
           }
         ];
-      };
+      }
+      // (lib.mapAttrs' genWorkerService cfg.workers.config);
 
     services.redis.servers.matrix-synapse = lib.mkIf cfg.configureRedisLocally {
       enable = true;

From 3a6a07ecf1b38dd887f7af4bec3da6fe3c8eb227 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Thu, 6 Jul 2023 23:04:11 +0200
Subject: [PATCH 05/16] nixos/synapse: automatically configure replication
 listener

---
 nixos/modules/services/matrix/synapse.nix | 52 ++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 2 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index b7591f984730..c779105abf25 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -525,7 +525,17 @@ in {
                   names = [ "federation" ];
                   compress = false;
                 }];
-              }];
+              }] ++ lib.optional hasWorkers {
+                port = 9093;
+                bind_addresses = [ "127.0.0.1" ];
+                type = "http";
+                tls = false;
+                x_forwarded = false;
+                resources = [{
+                  names = [ "replication" ];
+                  compress = false;
+                }];
+              };
               description = lib.mdDoc ''
                 List of ports that Synapse should listen on, their purpose and their configuration.
               '';
@@ -767,7 +777,18 @@ in {
               type = types.bool;
               default = false;
               description = lib.mdDoc ''
-                Whether to enable matrix synapse workers
+                Whether to enable matrix synapse workers.
+
+                ::: {.note}
+                  Enabling this will add a replication listener to the default
+                  value of `services.matrix-synapse.settings.listeners` and configure that
+                  listener as `services.matrix-synapse.settings.instance_map.main`.
+                  If you set either of those options, make sure to configure a replication
+                  listener yourself.
+
+                  A redis server is required for running workers. A local one can be enabled
+                  using `services.matrix-synapse.configureRedisLocally`.
+                :::
               '';
             };
             config = lib.mkOption {
@@ -843,12 +864,39 @@ in {
           automatically by setting `services.matrix-synapse.configureRedisLocally = true`.
         '';
       }
+      {
+        assertion =
+          let
+            main = cfg.settings.instance_map.main;
+            listener = lib.findFirst
+              (
+                listener:
+                  listener.port == main.port
+                  && (lib.any (resource: lib.any (name: name == "replication") resource.names) listener.resources)
+                  && (lib.any (bind: bind == main.host || bind == "0.0.0.0") listener.bind_addresses)
+              )
+              null
+              cfg.settings.listeners;
+          in
+          hasWorkers -> (listener != null);
+        message = ''
+          Workers for matrix-synapse require setting `services.matrix-synapse.settings.instance_map.main`
+          to any listener configured in `services.matrix-synapse.settings.listeners` with a `"replication"`
+          resource.
+
+          This is done by default unless you manually configure either of those settings.
+        '';
+      }
     ];
 
     services.matrix-synapse.settings.redis = lib.mkIf cfg.configureRedisLocally {
       enabled = true;
       path = config.services.redis.servers.matrix-synapse.unixSocket;
     };
+    services.matrix-synapse.settings.instance_map.main = lib.mkIf hasWorkers (lib.mkDefault {
+      host = "127.0.0.1";
+      port = 9093;
+    });
 
     services.matrix-synapse.configFile = configFile;
     services.matrix-synapse.package = wrapped;

From b20cbb12cdcb1fb81a22f58f028a2d876eafa832 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie.tauchert@relaxdays.de>
Date: Fri, 7 Jul 2023 11:27:55 +0200
Subject: [PATCH 06/16] nixos/synapse: add test for running synapse with
 workers

Co-authored-by: Daniel Olsen <daniel.olsen99@gmail.com>
---
 nixos/tests/all-tests.nix              |  1 +
 nixos/tests/matrix/synapse-workers.nix | 55 ++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 nixos/tests/matrix/synapse-workers.nix

diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 2d9674e69b64..0574c1db8754 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -468,6 +468,7 @@ in {
   matrix-appservice-irc = handleTest ./matrix/appservice-irc.nix {};
   matrix-conduit = handleTest ./matrix/conduit.nix {};
   matrix-synapse = handleTest ./matrix/synapse.nix {};
+  matrix-synapse-workers = handleTest ./matrix/synapse-workers.nix {};
   mattermost = handleTest ./mattermost.nix {};
   mediamtx = handleTest ./mediamtx.nix {};
   mediatomb = handleTest ./mediatomb.nix {};
diff --git a/nixos/tests/matrix/synapse-workers.nix b/nixos/tests/matrix/synapse-workers.nix
new file mode 100644
index 000000000000..a08b326abe62
--- /dev/null
+++ b/nixos/tests/matrix/synapse-workers.nix
@@ -0,0 +1,55 @@
+import ../make-test-python.nix ({ pkgs, ... }: {
+  name = "matrix-synapse-workers";
+  meta = with pkgs.lib; {
+    maintainers = teams.matrix.members;
+  };
+
+  nodes = {
+    homeserver =
+      { pkgs
+      , nodes
+      , ...
+      }: {
+        services.postgresql = {
+          enable = true;
+          initialScript = pkgs.writeText "synapse-init.sql" ''
+            CREATE ROLE "matrix-synapse" WITH LOGIN PASSWORD 'synapse';
+            CREATE DATABASE "matrix-synapse" WITH OWNER "matrix-synapse"
+            TEMPLATE template0
+            LC_COLLATE = "C"
+            LC_CTYPE = "C";
+          '';
+        };
+
+        services.matrix-synapse = {
+          enable = true;
+          settings = {
+            database = {
+              name = "psycopg2";
+              args.password = "synapse";
+            };
+            enable_registration = true;
+            enable_registration_without_verification = true;
+
+            federation_sender_instances = [ "federation_sender" ];
+          };
+          configureRedisLocally = true;
+          workers = {
+            enable = true;
+            config = {
+              "federation_sender" = {
+                worker_app = "synapse.app.generic_worker";
+              };
+            };
+          };
+        };
+      };
+  };
+
+  testScript = ''
+    start_all()
+
+    homeserver.wait_for_unit("matrix-synapse.service");
+    homeserver.wait_for_unit("matrix-synapse-worker-federation_sender.service");
+  '';
+})

From 2edea7611bff66697e1aa5f5b8921a5be9d3e6a6 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie.tauchert@relaxdays.de>
Date: Fri, 7 Jul 2023 11:55:10 +0200
Subject: [PATCH 07/16] nixos/synapse: document options better

Co-authored-by: Daniel Olsen <daniel.olsen99@gmail.com>
---
 nixos/modules/services/matrix/synapse.nix | 49 +++++++++++++++++++++++
 nixos/tests/matrix/synapse-workers.nix    |  4 +-
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index c779105abf25..52074f1d9a62 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -761,6 +761,28 @@ in {
               '';
             };
 
+            redis = lib.mkOption {
+              type = types.submodule {
+                freeformType = format.type;
+                options = {
+                  enabled = lib.mkOption {
+                    type = types.bool;
+                    default = false;
+                    description = lib.mdDoc ''
+                      Whether to use redis support
+                    '';
+                  };
+                };
+              };
+              default = { };
+              description = lib.mdDoc ''
+                Redis configuration for synapse.
+
+                See the
+                [upstream documentation](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/usage/configuration/config_documentation.md#redis)
+                for available options.
+              '';
+            };
           };
         };
       };
@@ -795,6 +817,14 @@ in {
               type = types.attrsOf (types.submodule {
                 freeformType = format.type;
                 options = {
+                  worker_app = lib.mkOption {
+                    type = types.enum [
+                      "synapse.app.generic_worker"
+                      "synapse.app.media_repository"
+                    ];
+                    description = "Type of this worker";
+                    default = "synapse.app.generic_worker";
+                  };
                   worker_listeners = lib.mkOption {
                     default = [ ];
                     type = types.listOf listenerType;
@@ -810,6 +840,25 @@ in {
                 [worker documention](https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration)
                 for possible values.
               '';
+              example = lib.literalExpression ''
+                {
+                  "federation_sender" = { };
+                  "federation_receiver" = {
+                    worker_listeners = [
+                      {
+                        type = "http";
+                        port = 8009;
+                        bind_addresses = [ "127.0.0.1" ];
+                        tls = false;
+                        x_forwarded = true;
+                        resources = [{
+                          names = [ "federation" ];
+                        }];
+                      }
+                    ];
+                  };
+                }
+              '';
             };
           };
         };
diff --git a/nixos/tests/matrix/synapse-workers.nix b/nixos/tests/matrix/synapse-workers.nix
index a08b326abe62..fb722037c3e2 100644
--- a/nixos/tests/matrix/synapse-workers.nix
+++ b/nixos/tests/matrix/synapse-workers.nix
@@ -37,9 +37,7 @@ import ../make-test-python.nix ({ pkgs, ... }: {
           workers = {
             enable = true;
             config = {
-              "federation_sender" = {
-                worker_app = "synapse.app.generic_worker";
-              };
+              "federation_sender" = { };
             };
           };
         };

From 857b4932eca927df0f2cb1ac8bbbae72a0960c0b Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie.tauchert@relaxdays.de>
Date: Mon, 31 Jul 2023 10:35:13 +0200
Subject: [PATCH 08/16] nixos/synapse: remove obsolete log context

see https://github.com/matrix-org/synapse/commit/0304ad0c3d79e44e78f9658e71f1e1533e3aa4e2 for when this was removed upstream
---
 nixos/modules/services/matrix/synapse-log_config.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse-log_config.yaml b/nixos/modules/services/matrix/synapse-log_config.yaml
index d85bdd1208f9..c4b2b0d8acf5 100644
--- a/nixos/modules/services/matrix/synapse-log_config.yaml
+++ b/nixos/modules/services/matrix/synapse-log_config.yaml
@@ -6,16 +6,10 @@ formatters:
     journal_fmt:
         format: '%(name)s: [%(request)s] %(message)s'
 
-filters:
-    context:
-        (): synapse.util.logcontext.LoggingContextFilter
-        request: ""
-
 handlers:
     journal:
         class: systemd.journal.JournalHandler
         formatter: journal_fmt
-        filters: [context]
         SYSLOG_IDENTIFIER: synapse
 
 root:

From 53ab84cf49a6146de35f31f81960185ff6075d55 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie.tauchert@relaxdays.de>
Date: Mon, 31 Jul 2023 11:34:56 +0200
Subject: [PATCH 09/16] nixos/synapse: automatically configure logging for
 synapse and workers

---
 .../services/matrix/synapse-log_config.yaml   | 19 ------
 nixos/modules/services/matrix/synapse.nix     | 62 +++++++++++++++++--
 2 files changed, 58 insertions(+), 23 deletions(-)
 delete mode 100644 nixos/modules/services/matrix/synapse-log_config.yaml

diff --git a/nixos/modules/services/matrix/synapse-log_config.yaml b/nixos/modules/services/matrix/synapse-log_config.yaml
deleted file mode 100644
index c4b2b0d8acf5..000000000000
--- a/nixos/modules/services/matrix/synapse-log_config.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-version: 1
-
-# In systemd's journal, loglevel is implicitly stored, so let's omit it
-# from the message text.
-formatters:
-    journal_fmt:
-        format: '%(name)s: [%(request)s] %(message)s'
-
-handlers:
-    journal:
-        class: systemd.journal.JournalHandler
-        formatter: journal_fmt
-        SYSLOG_IDENTIFIER: synapse
-
-root:
-    level: INFO
-    handlers: [journal]
-
-disable_existing_loggers: False
diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index 52074f1d9a62..f7ae3fa89536 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -69,6 +69,48 @@ let
     extras = wantedExtras;
     inherit (cfg) plugins;
   };
+
+  logConfig = logName: {
+    version = 1;
+    formatters.journal_fmt.format = "%(name)s: [%(request)s] %(message)s";
+    handlers.journal = {
+      class = "systemd.journal.JournalHandler";
+      formatter = "journal_fmt";
+      SYSLOG_IDENTIFIER = logName;
+    };
+    root = {
+      level = "INFO";
+      handlers = [ "journal" ];
+    };
+    disable_existing_loggers = false;
+  };
+  logConfigText = logName:
+    let
+      expr = ''
+        {
+          version = 1;
+          formatters.journal_fmt.format = "%(name)s: [%(request)s] %(message)s";
+          handlers.journal = {
+            class = "systemd.journal.JournalHandler";
+            formatter = "journal_fmt";
+            SYSLOG_IDENTIFIER = "${logName}";
+          };
+          root = {
+            level = "INFO";
+            handlers = [ "journal" ];
+          };
+          disable_existing_loggers = false;
+        };
+      '';
+    in
+    lib.literalMD ''
+      Path to a yaml file generated from this Nix expression:
+
+      ```
+      ${expr}
+      ```
+    '';
+  genLogConfigFile = logName: format.generate "synapse-log-${logName}.yaml" (logConfig logName);
 in {
 
   imports = [
@@ -448,8 +490,8 @@ in {
 
             log_config = mkOption {
               type = types.path;
-              default = ./synapse-log_config.yaml;
-              defaultText = lib.literalExpression "nixos/modules/services/matrix/synapse-log_config.yaml";
+              default = genLogConfigFile "synapse";
+              defaultText = logConfigText "synapse";
               description = lib.mdDoc ''
                 The file that holds the logging configuration.
               '';
@@ -814,7 +856,7 @@ in {
               '';
             };
             config = lib.mkOption {
-              type = types.attrsOf (types.submodule {
+              type = types.attrsOf (types.submodule ({name, ...}: {
                 freeformType = format.type;
                 options = {
                   worker_app = lib.mkOption {
@@ -832,8 +874,20 @@ in {
                       List of ports that this worker should listen on, their purpose and their configuration.
                     '';
                   };
+                  worker_log_config = lib.mkOption {
+                    type = types.path;
+                    default = genLogConfigFile "synapse-${name}";
+                    defaultText = logConfigText "synapse-${name}";
+                    description = lib.mdDoc ''
+                      The file for log configuration.
+
+                      See the [python documentation](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema)
+                      for the schema and the [upstream repository](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_log_config.yaml)
+                      for an example.
+                    '';
+                  };
                 };
-              });
+              }));
               default = { };
               description = lib.mdDoc ''
                 List of workers to configure. See the

From ca1ffe586948c3e5446387fe15ee241d0cd153ec Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Sun, 10 Sep 2023 15:35:59 +0200
Subject: [PATCH 10/16] nixos/synapse: move
 services.matrix-synapse.workers.config to services.matrix-synapse.workers

---
 nixos/modules/services/matrix/synapse.nix | 138 ++++++++++------------
 nixos/tests/matrix/synapse-workers.nix    |   5 +-
 2 files changed, 62 insertions(+), 81 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index f7ae3fa89536..25d3e3dcd1b9 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -13,7 +13,7 @@ let
   usePostgresql = cfg.settings.database.name == "psycopg2";
   hasLocalPostgresDB = let args = cfg.settings.database.args; in
     usePostgresql && (!(args ? host) || (elem args.host [ "localhost" "127.0.0.1" "::1" ]));
-  hasWorkers = cfg.workers.enable && (cfg.workers.config != { });
+  hasWorkers = cfg.workers != { };
 
   registerNewMatrixUser =
     let
@@ -832,90 +832,74 @@ in {
       workers = lib.mkOption {
         default = { };
         description = lib.mdDoc ''
-          Options for configuring workers. See `services.matrix-synapse.workers.enable`
-          for a more detailed description.
+          Options for configuring workers. Worker support will be enabled if at least one worker is configured here.
+
+          See the [worker documention](https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration)
+          for possible options for each worker. Worker-specific options overriding the shared homeserver configuration can be
+          specified here for each worker.
+
+          ::: {.note}
+            Worker support will add a replication listener to the default
+            value of [`services.matrix-synapse.settings.listeners`](#opt-services.matrix-synapse.settings.listeners) and configure that
+            listener as `services.matrix-synapse.settings.instance_map.main`.
+            If you set either of those options, make sure to configure a replication listener yourself.
+
+            A redis server is required for running workers. A local one can be enabled
+            using [`services.matrix-synapse.configureRedisLocally`](#opt-services.matrix-synapse.configureRedisLocally).
+          :::
         '';
-        type = types.submodule {
+        type = types.attrsOf (types.submodule ({name, ...}: {
+          freeformType = format.type;
           options = {
-            enable = lib.mkOption {
-              type = types.bool;
-              default = false;
+            worker_app = lib.mkOption {
+              type = types.enum [
+                "synapse.app.generic_worker"
+                "synapse.app.media_repository"
+              ];
+              description = "Type of this worker";
+              default = "synapse.app.generic_worker";
+            };
+            worker_listeners = lib.mkOption {
+              default = [ ];
+              type = types.listOf listenerType;
               description = lib.mdDoc ''
-                Whether to enable matrix synapse workers.
-
-                ::: {.note}
-                  Enabling this will add a replication listener to the default
-                  value of `services.matrix-synapse.settings.listeners` and configure that
-                  listener as `services.matrix-synapse.settings.instance_map.main`.
-                  If you set either of those options, make sure to configure a replication
-                  listener yourself.
-
-                  A redis server is required for running workers. A local one can be enabled
-                  using `services.matrix-synapse.configureRedisLocally`.
-                :::
+                List of ports that this worker should listen on, their purpose and their configuration.
               '';
             };
-            config = lib.mkOption {
-              type = types.attrsOf (types.submodule ({name, ...}: {
-                freeformType = format.type;
-                options = {
-                  worker_app = lib.mkOption {
-                    type = types.enum [
-                      "synapse.app.generic_worker"
-                      "synapse.app.media_repository"
-                    ];
-                    description = "Type of this worker";
-                    default = "synapse.app.generic_worker";
-                  };
-                  worker_listeners = lib.mkOption {
-                    default = [ ];
-                    type = types.listOf listenerType;
-                    description = lib.mdDoc ''
-                      List of ports that this worker should listen on, their purpose and their configuration.
-                    '';
-                  };
-                  worker_log_config = lib.mkOption {
-                    type = types.path;
-                    default = genLogConfigFile "synapse-${name}";
-                    defaultText = logConfigText "synapse-${name}";
-                    description = lib.mdDoc ''
-                      The file for log configuration.
-
-                      See the [python documentation](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema)
-                      for the schema and the [upstream repository](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_log_config.yaml)
-                      for an example.
-                    '';
-                  };
-                };
-              }));
-              default = { };
+            worker_log_config = lib.mkOption {
+              type = types.path;
+              default = genLogConfigFile "synapse-${name}";
+              defaultText = logConfigText "synapse-${name}";
               description = lib.mdDoc ''
-                List of workers to configure. See the
-                [worker documention](https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration)
-                for possible values.
-              '';
-              example = lib.literalExpression ''
-                {
-                  "federation_sender" = { };
-                  "federation_receiver" = {
-                    worker_listeners = [
-                      {
-                        type = "http";
-                        port = 8009;
-                        bind_addresses = [ "127.0.0.1" ];
-                        tls = false;
-                        x_forwarded = true;
-                        resources = [{
-                          names = [ "federation" ];
-                        }];
-                      }
-                    ];
-                  };
-                }
+                The file for log configuration.
+
+                See the [python documentation](https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema)
+                for the schema and the [upstream repository](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_log_config.yaml)
+                for an example.
               '';
             };
           };
-        };
+        }));
+        default = { };
+        example = lib.literalExpression ''
+          {
+            "federation_sender" = { };
+            "federation_receiver" = {
+              worker_listeners = [
+                {
+                  type = "http";
+                  port = 8009;
+                  bind_addresses = [ "127.0.0.1" ];
+                  tls = false;
+                  x_forwarded = true;
+                  resources = [{
+                    names = [ "federation" ];
+                  }];
+                }
+              ];
+            };
+          }
+        '';
       };
 
       extraConfigFiles = mkOption {
@@ -1131,7 +1115,7 @@ in {
           }
         ];
       }
-      // (lib.mapAttrs' genWorkerService cfg.workers.config);
+      // (lib.mapAttrs' genWorkerService cfg.workers);
 
     services.redis.servers.matrix-synapse = lib.mkIf cfg.configureRedisLocally {
       enable = true;
diff --git a/nixos/tests/matrix/synapse-workers.nix b/nixos/tests/matrix/synapse-workers.nix
index fb722037c3e2..e90301aeae9e 100644
--- a/nixos/tests/matrix/synapse-workers.nix
+++ b/nixos/tests/matrix/synapse-workers.nix
@@ -35,10 +35,7 @@ import ../make-test-python.nix ({ pkgs, ... }: {
           };
           configureRedisLocally = true;
           workers = {
-            enable = true;
-            config = {
-              "federation_sender" = { };
-            };
+            "federation_sender" = { };
           };
         };
       };

From c693c2fd963b7a4b94958c471baed36bfe563879 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Sun, 10 Sep 2023 15:36:40 +0200
Subject: [PATCH 11/16] nixos/synapse: simplify replication listener assertion

---
 nixos/modules/services/matrix/synapse.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index 25d3e3dcd1b9..a90054314863 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -959,8 +959,8 @@ in {
               (
                 listener:
                   listener.port == main.port
-                  && (lib.any (resource: lib.any (name: name == "replication") resource.names) listener.resources)
-                  && (lib.any (bind: bind == main.host || bind == "0.0.0.0") listener.bind_addresses)
+                  && (lib.any (resource: builtins.elem "replication" resource.names) listener.resources)
+                  && (lib.any (bind: bind == main.host || bind == "0.0.0.0" || bind == "::") listener.bind_addresses)
               )
               null
               cfg.settings.listeners;

From dea34ad0fa3ecd3bed440b9b0a8e4b57a540583a Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Mon, 11 Sep 2023 10:03:57 +0200
Subject: [PATCH 12/16] nixos/synapse: default tls to off for workers and
 document worker replication port

---
 nixos/modules/services/matrix/synapse.nix | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index a90054314863..e49c0d12a4f4 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -198,7 +198,7 @@ in {
   ];
 
   options = let
-    listenerType = types.submodule {
+    listenerType = workerContext: types.submodule {
       options = {
         port = mkOption {
           type = types.port;
@@ -241,7 +241,7 @@ in {
 
         tls = mkOption {
           type = types.bool;
-          default = true;
+          default = !workerContext;
           example = false;
           description = lib.mdDoc ''
             Whether to enable TLS on the listener socket.
@@ -553,7 +553,7 @@ in {
             };
 
             listeners = mkOption {
-              type = types.listOf listenerType;
+              type = types.listOf (listenerType false);
               default = [{
                 port = 8008;
                 bind_addresses = [ "127.0.0.1" ];
@@ -580,6 +580,10 @@ in {
               };
               description = lib.mdDoc ''
                 List of ports that Synapse should listen on, their purpose and their configuration.
+
+                By default, synapse will be configured for client and federation traffic on port 8008, and
+                for worker replication traffic on port 9093. See [`services.matrix-synapse.workers`](#opt-services.matrix-synapse.workers)
+                for more details.
               '';
             };
 
@@ -839,7 +843,7 @@ in {
           specified here for each worker.
 
           ::: {.note}
-            Worker support will add a replication listener to the default
+            Worker support will add a replication listener on port 9093 to the main synapse process using the default
             value of [`services.matrix-synapse.settings.listeners`](#opt-services.matrix-synapse.settings.listeners) and configure that
             listener as `services.matrix-synapse.settings.instance_map.main`.
             If you set either of those options, make sure to configure a replication listener yourself.
@@ -861,7 +865,7 @@ in {
             };
             worker_listeners = lib.mkOption {
               default = [ ];
-              type = types.listOf listenerType;
+              type = types.listOf (listenerType true);
               description = lib.mdDoc ''
                 List of ports that this worker should listen on, their purpose and their configuration.
               '';

From 99e7130d69320e77bed8a71710f656b6a0a7f84e Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Mon, 11 Sep 2023 10:12:06 +0200
Subject: [PATCH 13/16] matrix-synapse: add worker test to passthru.tests

---
 pkgs/servers/matrix-synapse/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/servers/matrix-synapse/default.nix b/pkgs/servers/matrix-synapse/default.nix
index 478be3129e4d..60da7eafa379 100644
--- a/pkgs/servers/matrix-synapse/default.nix
+++ b/pkgs/servers/matrix-synapse/default.nix
@@ -157,7 +157,7 @@ python3.pkgs.buildPythonApplication rec {
   '';
 
   passthru = {
-    tests = { inherit (nixosTests) matrix-synapse; };
+    tests = { inherit (nixosTests) matrix-synapse matrix-synapse-workers; };
     inherit plugins tools;
     python = python3;
   };

From 6b95c618e231487a3cd105b59d3f7df199abcd4b Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Mon, 11 Sep 2023 15:58:01 +0200
Subject: [PATCH 14/16] nixos/rl-2311: fix option references for synapse
 workers

---
 nixos/doc/manual/release-notes/rl-2311.section.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md
index 24aaf9d79ebb..58d98b0f0ca4 100644
--- a/nixos/doc/manual/release-notes/rl-2311.section.md
+++ b/nixos/doc/manual/release-notes/rl-2311.section.md
@@ -242,7 +242,7 @@
 Unfortunately all servers supporting new clients (newer version of anki-sync-server, anki's built in sync server and this new rust package) do not support the older sync protocol that was used in the old server, so such old clients will also need updating and in particular the anki package in nixpkgs is also being updated in this release.
 The module update takes care of the new config syntax and the data itself (user login and cards) are compatible, so users of the module will be able to just log in again after updating both client and server without any extra action.
 
-- `services.matrix-synapse` has new options to configure worker processes for matrix-synapse using `services.matrix-synapse.workers.enable` and `services.matrix-synapse.workers.config`. It's also now possible to configure a local redis server using `services.matrix-synapse.configureRedisLocally`.
+- `services.matrix-synapse` has new options to configure worker processes for matrix-synapse using [`services.matrix-synapse.workers`](#opt-services.matrix-synapse.workers). It's also now possible to configure a local redis server using [`services.matrix-synapse.configureRedisLocally`](#opt-services.matrix-synapse.configureRedisLocally).
 
 - `services.nginx` gained a `defaultListen` option at server-level with support for PROXY protocol listeners, also `proxyProtocol` is now exposed in `services.nginx.virtualHosts.<name>.listen` option. It is now possible to run PROXY listeners and non-PROXY listeners at a server-level, see [#213510](https://github.com/NixOS/nixpkgs/pull/213510/) for more details.
 

From aed8a5c6cd849a8fa819ccdbd6877915227d500a Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie.tauchert@relaxdays.de>
Date: Thu, 14 Sep 2023 09:34:34 +0200
Subject: [PATCH 15/16] nixos/synapse: add documentation for required reverse
 proxy setup

---
 nixos/modules/services/matrix/synapse.nix | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index e49c0d12a4f4..0ece1cb48ce0 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -850,6 +850,12 @@ in {
 
             A redis server is required for running workers. A local one can be enabled
             using [`services.matrix-synapse.configureRedisLocally`](#opt-services.matrix-synapse.configureRedisLocally).
+
+            Workers also require a proper reverse proxy setup to direct incoming requests to the appropriate process. See
+            the [reverse proxy documentation](https://matrix-org.github.io/synapse/latest/reverse_proxy.html) for a
+            general reverse proxying setup and
+            the [worker documentation](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications)
+            for the available endpoints per worker application.
           :::
         '';
         type = types.attrsOf (types.submodule ({name, ...}: {

From 24f6a70abfa36d9a6c604bd324e4509e92777bc4 Mon Sep 17 00:00:00 2001
From: Sophie Tauchert <sophie@999eagle.moe>
Date: Mon, 18 Sep 2023 10:52:52 +0200
Subject: [PATCH 16/16] nixos/synapse: make sure workers require main process

This should ensure systemd handles starting all services (main and
workers) in a single transaction, thus preserving unit orderings
defined through After= even when not restarting the target.
---
 nixos/modules/services/matrix/synapse.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix
index 0ece1cb48ce0..554e9ca2ecc3 100644
--- a/nixos/modules/services/matrix/synapse.nix
+++ b/nixos/modules/services/matrix/synapse.nix
@@ -1087,6 +1087,7 @@ in {
                 description = "Synapse Matrix worker ${name}";
                 # make sure the main process starts first for potential database migrations
                 after = [ "matrix-synapse.service" ];
+                requires = [ "matrix-synapse.service" ];
                 serviceConfig = {
                   ExecStart = ''
                     ${cfg.package}/bin/synapse_worker \