From ccb70a892af119fe3933ffaee852020ace79e7c9 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:25:02 +0200 Subject: [PATCH 1/6] python3Packages.mailsuite: Init at 1.6.1 --- .../python-modules/mailsuite/default.nix | 39 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 41 insertions(+) create mode 100644 pkgs/development/python-modules/mailsuite/default.nix diff --git a/pkgs/development/python-modules/mailsuite/default.nix b/pkgs/development/python-modules/mailsuite/default.nix new file mode 100644 index 000000000000..23167af45b0e --- /dev/null +++ b/pkgs/development/python-modules/mailsuite/default.nix @@ -0,0 +1,39 @@ +{ buildPythonPackage +, fetchPypi +, pythonOlder +, lib + +# pythonPackages +, dnspython +, html2text +, mail-parser +, IMAPClient +}: + +buildPythonPackage rec { + pname = "mailsuite"; + version = "1.6.1"; + + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + sha256 = "17bsnfjjzv8hx5h397p5pa92l6cqc53i0zjjz2p7bjj3xqzhs45a"; + }; + + propagatedBuildInputs = [ + dnspython + html2text + mail-parser + IMAPClient + ]; + + pythonImportsCheck = [ "mailsuite" ]; + + meta = { + description = "A Python package to simplify receiving, parsing, and sending email"; + homepage = "https://seanthegeek.github.io/mailsuite/"; + maintainers = with lib.maintainers; [ talyz ]; + license = lib.licenses.asl20; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8d00d957f7ce..d2978ce0d824 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4397,6 +4397,8 @@ in { makefun = callPackage ../development/python-modules/makefun { }; + mailsuite = callPackage ../development/python-modules/mailsuite { }; + Mako = callPackage ../development/python-modules/Mako { }; malduck= callPackage ../development/python-modules/malduck { }; From 5c61b9f3cb6ea038efc1390bd26f1f65f135c717 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:29:36 +0200 Subject: [PATCH 2/6] python3Packages.parsedmarc: Init at 7.0.1 --- .../python-modules/parsedmarc/default.nix | 72 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 74 insertions(+) create mode 100644 pkgs/development/python-modules/parsedmarc/default.nix diff --git a/pkgs/development/python-modules/parsedmarc/default.nix b/pkgs/development/python-modules/parsedmarc/default.nix new file mode 100644 index 000000000000..a19c00b004a6 --- /dev/null +++ b/pkgs/development/python-modules/parsedmarc/default.nix @@ -0,0 +1,72 @@ +{ buildPythonPackage +, fetchPypi +, fetchurl +, pythonOlder +, lib + +# pythonPackages +, tqdm +, dnspython +, expiringdict +, urllib3 +, requests +, publicsuffix2 +, xmltodict +, geoip2 +, IMAPClient +, dateparser +, elasticsearch-dsl +, kafka-python +, mailsuite +, lxml +, boto3 +}: + +let + dashboard = fetchurl { + url = "https://raw.githubusercontent.com/domainaware/parsedmarc/77331b55c54cb3269205295bd57d0ab680638964/grafana/Grafana-DMARC_Reports.json"; + sha256 = "0wbihyqbb4ndjg79qs8088zgrcg88km8khjhv2474y7nzjzkf43i"; + }; +in +buildPythonPackage rec { + pname = "parsedmarc"; + version = "7.0.1"; + + disabled = pythonOlder "3.7"; + + src = fetchPypi { + inherit pname version; + sha256 = "1mi4hx410y7ikpfy1582lm252si0c3yryj0idqgqbx417fm21jjc"; + }; + + propagatedBuildInputs = [ + tqdm + dnspython + expiringdict + urllib3 + requests + publicsuffix2 + xmltodict + geoip2 + IMAPClient + dateparser + elasticsearch-dsl + kafka-python + mailsuite + lxml + boto3 + ]; + + pythonImportsCheck = [ "parsedmarc" ]; + + passthru = { + inherit dashboard; + }; + + meta = { + description = "Python module and CLI utility for parsing DMARC reports"; + homepage = "https://domainaware.github.io/parsedmarc/"; + maintainers = with lib.maintainers; [ talyz ]; + license = lib.licenses.asl20; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d2978ce0d824..05a811e60f6a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -5255,6 +5255,8 @@ in { parsedatetime = callPackage ../development/python-modules/parsedatetime { }; + parsedmarc = callPackage ../development/python-modules/parsedmarc { }; + parsel = callPackage ../development/python-modules/parsel { }; parse-type = callPackage ../development/python-modules/parse-type { }; From 3a1e1f062432fbc867502187972b37f7a6f9fe63 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:33:13 +0200 Subject: [PATCH 3/6] nixos/elasticsearch: Wait for elasticsearch to start up properly Other services that depend on elasticsearch should be started after it, but since the versions we're packaging have to run as "Type=simple", they're started as soon as the elasticsearch binary has been executed, likely winning the race against it. This makes sure elasticsearch is up and running, responding to a simple query, before dependents are started. --- nixos/modules/services/search/elasticsearch.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index 1d7a28d5d245..3f676db27523 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -201,6 +201,13 @@ in if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi ''; + postStart = '' + # Make sure elasticsearch is up and running before dependents + # are started + while ! ${pkgs.curl}/bin/curl -sS -f http://localhost:${toString cfg.port} 2>/dev/null; do + sleep 1 + done + ''; }; environment.systemPackages = [ cfg.package ]; From 98d9617705a2b04b12b25adc7037e798b225c1b6 Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 14:32:20 +0200 Subject: [PATCH 4/6] nixos/parsedmarc: Add NixOS module --- nixos/modules/module-list.nix | 1 + .../services/monitoring/parsedmarc.nix | 532 ++++++++++++++++++ 2 files changed, 533 insertions(+) create mode 100644 nixos/modules/services/monitoring/parsedmarc.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index fed212c0332c..c01c55b26700 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -621,6 +621,7 @@ ./services/monitoring/munin.nix ./services/monitoring/nagios.nix ./services/monitoring/netdata.nix + ./services/monitoring/parsedmarc.nix ./services/monitoring/prometheus/default.nix ./services/monitoring/prometheus/alertmanager.nix ./services/monitoring/prometheus/exporters.nix diff --git a/nixos/modules/services/monitoring/parsedmarc.nix b/nixos/modules/services/monitoring/parsedmarc.nix new file mode 100644 index 000000000000..58137a74d395 --- /dev/null +++ b/nixos/modules/services/monitoring/parsedmarc.nix @@ -0,0 +1,532 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.parsedmarc; + ini = pkgs.formats.ini {}; +in +{ + options.services.parsedmarc = { + + enable = lib.mkEnableOption '' + parsedmarc, a DMARC report monitoring service + ''; + + provision = { + localMail = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether Postfix and Dovecot should be set up to receive + mail locally. parsedmarc will be configured to watch the + local inbox as the automatically created user specified in + + ''; + }; + + recipientName = lib.mkOption { + type = lib.types.str; + default = "dmarc"; + description = '' + The DMARC mail recipient name, i.e. the name part of the + email address which receives DMARC reports. + + A local user with this name will be set up and assigned a + randomized password on service start. + ''; + }; + + hostname = lib.mkOption { + type = lib.types.str; + default = config.networking.fqdn; + defaultText = "config.networking.fqdn"; + example = "monitoring.example.com"; + description = '' + The hostname to use when configuring Postfix. + + Should correspond to the host's fully qualified domain + name and the domain part of the email address which + receives DMARC reports. You also have to set up an MX record + pointing to this domain name. + ''; + }; + }; + + geoIp = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to enable and configure the geoipupdate + service to automatically fetch GeoIP databases. Not crucial, + but recommended for full functionality. + + To finish the setup, you need to manually set the and + + options. + ''; + }; + + elasticsearch = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Whether to set up and use a local instance of Elasticsearch. + ''; + }; + + grafana = { + datasource = lib.mkOption { + type = lib.types.bool; + default = cfg.provision.elasticsearch && config.services.grafana.enable; + apply = x: x && cfg.provision.elasticsearch; + description = '' + Whether the automatically provisioned Elasticsearch + instance should be added as a grafana datasource. Has no + effect unless + + is also enabled. + ''; + }; + + dashboard = lib.mkOption { + type = lib.types.bool; + default = config.services.grafana.enable; + description = '' + Whether the official parsedmarc grafana dashboard should + be provisioned to the local grafana instance. + ''; + }; + }; + }; + + settings = lib.mkOption { + description = '' + Configuration parameters to set in + parsedmarc.ini. For a full list of + available parameters, see + . + ''; + + type = lib.types.submodule { + freeformType = ini.type; + + options = { + general = { + save_aggregate = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Save aggregate report data to Elasticsearch and/or Splunk. + ''; + }; + + save_forensic = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Save forensic report data to Elasticsearch and/or Splunk. + ''; + }; + }; + + imap = { + host = lib.mkOption { + type = lib.types.str; + default = "localhost"; + description = '' + The IMAP server hostname or IP address. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 993; + description = '' + The IMAP server port. + ''; + }; + + ssl = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use an encrypted SSL/TLS connection. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The IMAP server username. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the IMAP server password. + ''; + }; + + watch = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use the IMAP IDLE command to process messages as they arrive. + ''; + }; + + delete = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Delete messages after processing them, instead of archiving them. + ''; + }; + }; + + smtp = { + host = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The SMTP server hostname or IP address. + ''; + }; + + port = lib.mkOption { + type = with lib.types; nullOr port; + default = null; + description = '' + The SMTP server port. + ''; + }; + + ssl = lib.mkOption { + type = with lib.types; nullOr bool; + default = null; + description = '' + Use an encrypted SSL/TLS connection. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The SMTP server username. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the SMTP server password. + ''; + }; + + from = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + The From address to use for the + outgoing mail. + ''; + }; + + to = lib.mkOption { + type = with lib.types; nullOr (listOf str); + default = null; + description = '' + The addresses to send outgoing mail to. + ''; + }; + }; + + elasticsearch = { + hosts = lib.mkOption { + default = []; + type = with lib.types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + description = '' + A list of Elasticsearch hosts to push parsed reports + to. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + Username to use when connecting to Elasticsearch, if + required. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the password to use when + connecting to Elasticsearch, if required. + ''; + }; + + ssl = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to use an encrypted SSL/TLS connection. + ''; + }; + + cert_path = lib.mkOption { + type = lib.types.path; + default = "/etc/ssl/certs/ca-certificates.crt"; + description = '' + The path to a TLS certificate bundle used to verify + the server's certificate. + ''; + }; + }; + + kafka = { + hosts = lib.mkOption { + default = []; + type = with lib.types; listOf str; + apply = x: if x == [] then null else lib.concatStringsSep "," x; + description = '' + A list of Apache Kafka hosts to publish parsed reports + to. + ''; + }; + + user = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + Username to use when connecting to Kafka, if + required. + ''; + }; + + password = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + The path to a file containing the password to use when + connecting to Kafka, if required. + ''; + }; + + ssl = lib.mkOption { + type = with lib.types; nullOr bool; + default = null; + description = '' + Whether to use an encrypted SSL/TLS connection. + ''; + }; + + aggregate_topic = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "aggregate"; + description = '' + The Kafka topic to publish aggregate reports on. + ''; + }; + + forensic_topic = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "forensic"; + description = '' + The Kafka topic to publish forensic reports on. + ''; + }; + }; + + }; + + }; + }; + + }; + + config = lib.mkIf cfg.enable { + + services.elasticsearch.enable = lib.mkDefault cfg.provision.elasticsearch; + + services.geoipupdate = lib.mkIf cfg.provision.geoIp { + enable = true; + settings = { + EditionIDs = [ + "GeoLite2-ASN" + "GeoLite2-City" + "GeoLite2-Country" + ]; + DatabaseDirectory = "/var/lib/GeoIP"; + }; + }; + + services.dovecot2 = lib.mkIf cfg.provision.localMail.enable { + enable = true; + protocols = [ "imap" ]; + }; + + services.postfix = lib.mkIf cfg.provision.localMail.enable { + enable = true; + origin = cfg.provision.localMail.hostname; + config = { + myhostname = cfg.provision.localMail.hostname; + mydestination = cfg.provision.localMail.hostname; + }; + }; + + services.grafana = { + declarativePlugins = with pkgs.grafanaPlugins; + lib.mkIf cfg.provision.grafana.dashboard [ + grafana-worldmap-panel + grafana-piechart-panel + ]; + + provision = { + enable = cfg.provision.grafana.datasource || cfg.provision.grafana.dashboard; + datasources = + let + pkgVer = lib.getVersion config.services.elasticsearch.package; + esVersion = + if lib.versionOlder pkgVer "7" then + "60" + else if lib.versionOlder pkgVer "8" then + "70" + else + throw "When provisioning parsedmarc grafana datasources: unknown Elasticsearch version."; + in + lib.mkIf cfg.provision.grafana.datasource [ + { + name = "dmarc-ag"; + type = "elasticsearch"; + access = "proxy"; + url = "localhost:9200"; + jsonData = { + timeField = "date_range"; + inherit esVersion; + }; + } + { + name = "dmarc-fo"; + type = "elasticsearch"; + access = "proxy"; + url = "localhost:9200"; + jsonData = { + timeField = "date_range"; + inherit esVersion; + }; + } + ]; + dashboards = lib.mkIf cfg.provision.grafana.dashboard [{ + name = "parsedmarc"; + options.path = "${pkgs.python3Packages.parsedmarc.dashboard}"; + }]; + }; + }; + + services.parsedmarc.settings = lib.mkMerge [ + (lib.mkIf cfg.provision.elasticsearch { + elasticsearch = { + hosts = [ "localhost:9200" ]; + ssl = false; + }; + }) + (lib.mkIf cfg.provision.localMail.enable { + imap = { + host = "localhost"; + port = 143; + ssl = false; + user = cfg.provision.localMail.recipientName; + password = "${pkgs.writeText "imap-password" "@imap-password@"}"; + watch = true; + }; + }) + ]; + + systemd.services.parsedmarc = + let + # Remove any empty attributes from the config, i.e. empty + # lists, empty attrsets and null. This makes it possible to + # list interesting options in `settings` without them always + # ending up in the resulting config. + filteredConfig = lib.converge (lib.filterAttrsRecursive (_: v: ! builtins.elem v [ null [] {} ])) cfg.settings; + parsedmarcConfig = ini.generate "parsedmarc.ini" filteredConfig; + mkSecretReplacement = file: + lib.optionalString (file != null) '' + replace-secret '${file}' '${file}' /run/parsedmarc/parsedmarc.ini + ''; + in + { + wantedBy = [ "multi-user.target" ]; + after = [ "postfix.service" "dovecot2.service" "elasticsearch.service" ]; + path = with pkgs; [ replace-secret openssl shadow ]; + serviceConfig = { + ExecStartPre = let + startPreFullPrivileges = '' + set -o errexit -o pipefail -o nounset -o errtrace + shopt -s inherit_errexit + + umask u=rwx,g=,o= + cp ${parsedmarcConfig} /run/parsedmarc/parsedmarc.ini + chown parsedmarc:parsedmarc /run/parsedmarc/parsedmarc.ini + ${mkSecretReplacement cfg.settings.smtp.password} + ${mkSecretReplacement cfg.settings.imap.password} + ${mkSecretReplacement cfg.settings.elasticsearch.password} + ${mkSecretReplacement cfg.settings.kafka.password} + '' + lib.optionalString cfg.provision.localMail.enable '' + openssl rand -hex 64 >/run/parsedmarc/dmarc_user_passwd + replace-secret '@imap-password@' '/run/parsedmarc/dmarc_user_passwd' /run/parsedmarc/parsedmarc.ini + echo "Setting new randomized password for user '${cfg.provision.localMail.recipientName}'." + cat <(echo -n "${cfg.provision.localMail.recipientName}:") /run/parsedmarc/dmarc_user_passwd | chpasswd + ''; + in + "+${pkgs.writeShellScript "parsedmarc-start-pre-full-privileges" startPreFullPrivileges}"; + Type = "simple"; + User = "parsedmarc"; + Group = "parsedmarc"; + DynamicUser = true; + RuntimeDirectory = "parsedmarc"; + RuntimeDirectoryMode = 0700; + CapabilityBoundingSet = ""; + PrivateDevices = true; + PrivateMounts = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ]; + RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ]; + RestrictRealtime = true; + RestrictNamespaces = true; + MemoryDenyWriteExecute = true; + LockPersonality = true; + SystemCallArchitectures = "native"; + ExecStart = "${pkgs.python3Packages.parsedmarc}/bin/parsedmarc -c /run/parsedmarc/parsedmarc.ini"; + }; + }; + + users.users.${cfg.provision.localMail.recipientName} = lib.mkIf cfg.provision.localMail.enable { + isNormalUser = true; + description = "DMARC mail recipient"; + }; + }; +} From d46e78fd76f9b27bc3b8203dc16aa21371ea1bbd Mon Sep 17 00:00:00 2001 From: talyz Date: Wed, 2 Jun 2021 18:19:37 +0200 Subject: [PATCH 5/6] nixos/parsedmarc: Add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/parsedmarc/default.nix | 224 ++++++++++++++++++ .../python-modules/parsedmarc/default.nix | 2 + 3 files changed, 227 insertions(+) create mode 100644 nixos/tests/parsedmarc/default.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 6baa986b2bda..dab99dd8d043 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -336,6 +336,7 @@ in pam-u2f = handleTest ./pam-u2f.nix {}; pantheon = handleTest ./pantheon.nix {}; paperless-ng = handleTest ./paperless-ng.nix {}; + parsedmarc = handleTest ./parsedmarc {}; pdns-recursor = handleTest ./pdns-recursor.nix {}; peerflix = handleTest ./peerflix.nix {}; pgjwt = handleTest ./pgjwt.nix {}; diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix new file mode 100644 index 000000000000..d838d3b6a39c --- /dev/null +++ b/nixos/tests/parsedmarc/default.nix @@ -0,0 +1,224 @@ +# This tests parsedmarc by sending a report to its monitored email +# address and reading the results out of Elasticsearch. + +{ pkgs, ... }@args: +let + inherit (import ../../lib/testing-python.nix args) makeTest; + + dmarcTestReport = builtins.fetchurl { + name = "dmarc-test-report"; + url = "https://github.com/domainaware/parsedmarc/raw/f45ab94e0608088e0433557608d9f4e9517d3afe/samples/aggregate/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip"; + sha256 = "0dq64cj49711kbja27pjl2hy0d3azrjxg91kqrh40x46fkn1dwkx"; + }; + + sendEmail = address: + pkgs.writeScriptBin "send-email" '' + #!${pkgs.python3.interpreter} + import smtplib + from email import encoders + from email.mime.base import MIMEBase + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + + sender_email = "dmarc_tester@fake.domain" + receiver_email = "${address}" + + message = MIMEMultipart() + message["From"] = sender_email + message["To"] = receiver_email + message["Subject"] = "DMARC test" + + message.attach(MIMEText("Testing parsedmarc", "plain")) + + attachment = MIMEBase("application", "zip") + + with open("${dmarcTestReport}", "rb") as report: + attachment.set_payload(report.read()) + + encoders.encode_base64(attachment) + + attachment.add_header( + "Content-Disposition", + "attachment; filename= estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip", + ) + + message.attach(attachment) + text = message.as_string() + + with smtplib.SMTP('localhost') as server: + server.sendmail(sender_email, receiver_email, text) + server.quit() + ''; +in +{ + localMail = makeTest + { + name = "parsedmarc-local-mail"; + meta = with pkgs.lib.maintainers; { + maintainers = [ talyz ]; + }; + + nodes.parsedmarc = + { nodes, ... }: + { + virtualisation.memorySize = 2048; + + services.postfix = { + enableSubmission = true; + enableSubmissions = true; + submissionsOptions = { + smtpd_sasl_auth_enable = "yes"; + smtpd_client_restrictions = "permit"; + }; + }; + + services.parsedmarc = { + enable = true; + provision = { + geoIp = false; + localMail = { + enable = true; + hostname = "localhost"; + }; + }; + }; + + services.elasticsearch.package = pkgs.elasticsearch7-oss; + + environment.systemPackages = [ + (sendEmail "dmarc@localhost") + pkgs.jq + ]; + }; + + testScript = { nodes }: + let + esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + in '' + parsedmarc.start() + parsedmarc.wait_for_unit("postfix.service") + parsedmarc.wait_for_unit("dovecot2.service") + parsedmarc.wait_for_unit("parsedmarc.service") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}" + ) + + parsedmarc.fail( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + parsedmarc.succeed("send-email") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + ''; + }; + + externalMail = + let + certs = import ../common/acme/server/snakeoil-certs.nix; + mailDomain = certs.domain; + parsedmarcDomain = "parsedmarc.fake.domain"; + in + makeTest { + name = "parsedmarc-external-mail"; + meta = with pkgs.lib.maintainers; { + maintainers = [ talyz ]; + }; + + nodes = { + parsedmarc = + { nodes, ... }: + { + virtualisation.memorySize = 2048; + + security.pki.certificateFiles = [ + certs.ca.cert + ]; + + networking.extraHosts = '' + 127.0.0.1 ${parsedmarcDomain} + ${nodes.mail.config.networking.primaryIPAddress} ${mailDomain} + ''; + + services.parsedmarc = { + enable = true; + provision.geoIp = false; + settings.imap = { + host = mailDomain; + port = 993; + ssl = true; + user = "alice"; + password = "${pkgs.writeText "imap-password" "foobar"}"; + watch = true; + }; + }; + + services.elasticsearch.package = pkgs.elasticsearch7-oss; + + environment.systemPackages = [ + pkgs.jq + ]; + }; + + mail = + { nodes, ... }: + { + imports = [ ../common/user-account.nix ]; + + networking.extraHosts = '' + 127.0.0.1 ${mailDomain} + ${nodes.parsedmarc.config.networking.primaryIPAddress} ${parsedmarcDomain} + ''; + + services.dovecot2 = { + enable = true; + protocols = [ "imap" ]; + sslCACert = "${certs.ca.cert}"; + sslServerCert = "${certs.${mailDomain}.cert}"; + sslServerKey = "${certs.${mailDomain}.key}"; + }; + + services.postfix = { + enable = true; + origin = mailDomain; + config = { + myhostname = mailDomain; + mydestination = mailDomain; + }; + enableSubmission = true; + enableSubmissions = true; + submissionsOptions = { + smtpd_sasl_auth_enable = "yes"; + smtpd_client_restrictions = "permit"; + }; + }; + environment.systemPackages = [ (sendEmail "alice@${mailDomain}") ]; + + networking.firewall.allowedTCPPorts = [ 993 ]; + }; + }; + + testScript = { nodes }: + let + esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + in '' + mail.start() + mail.wait_for_unit("postfix.service") + mail.wait_for_unit("dovecot2.service") + + parsedmarc.start() + parsedmarc.wait_for_unit("parsedmarc.service") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}" + ) + + parsedmarc.fail( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + mail.succeed("send-email") + parsedmarc.wait_until_succeeds( + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + ) + ''; + }; +} diff --git a/pkgs/development/python-modules/parsedmarc/default.nix b/pkgs/development/python-modules/parsedmarc/default.nix index a19c00b004a6..b9fb35016c3b 100644 --- a/pkgs/development/python-modules/parsedmarc/default.nix +++ b/pkgs/development/python-modules/parsedmarc/default.nix @@ -3,6 +3,7 @@ , fetchurl , pythonOlder , lib +, nixosTests # pythonPackages , tqdm @@ -61,6 +62,7 @@ buildPythonPackage rec { passthru = { inherit dashboard; + tests = nixosTests.parsedmarc; }; meta = { From 6496902fb2b1496c7d5aba78afd4571584cd751e Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 3 Jun 2021 18:31:15 +0200 Subject: [PATCH 6/6] nixos/parsedmarc: Add manual entry and release note --- .../from_md/release-notes/rl-2111.section.xml | 11 ++ .../manual/release-notes/rl-2111.section.md | 5 + .../modules/services/monitoring/parsedmarc.md | 113 ++++++++++++++++ .../services/monitoring/parsedmarc.nix | 5 + .../services/monitoring/parsedmarc.xml | 125 ++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 nixos/modules/services/monitoring/parsedmarc.md create mode 100644 nixos/modules/services/monitoring/parsedmarc.xml diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index 57cd98db5894..6eaba9111a2b 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -239,6 +239,17 @@ programs.git. + + + parsedmarc, + a service which parses incoming + DMARC reports and + stores or sends them to a downstream service for further + analysis. Documented in + its manual + entry. + +
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 440d063abf28..b77bd30ea17c 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -73,6 +73,11 @@ subsonic-compatible api. Available as [navidrome](#opt-services.navidrome.enable - [git](https://git-scm.com), a distributed version control system. Available as [programs.git](options.html#opt-programs.git.enable). +- [parsedmarc](https://domainaware.github.io/parsedmarc/), a service + which parses incoming [DMARC](https://dmarc.org/) reports and stores + or sends them to a downstream service for further analysis. + Documented in [its manual entry](#module-services-parsedmarc). + ## Backward Incompatibilities {#sec-release-21.11-incompatibilities} diff --git a/nixos/modules/services/monitoring/parsedmarc.md b/nixos/modules/services/monitoring/parsedmarc.md new file mode 100644 index 000000000000..d93134a4cc76 --- /dev/null +++ b/nixos/modules/services/monitoring/parsedmarc.md @@ -0,0 +1,113 @@ +# parsedmarc {#module-services-parsedmarc} +[parsedmarc](https://domainaware.github.io/parsedmarc/) is a service +which parses incoming [DMARC](https://dmarc.org/) reports and stores +or sends them to a downstream service for further analysis. In +combination with Elasticsearch, Grafana and the included Grafana +dashboard, it provides a handy overview of DMARC reports over time. + +## Basic usage {#module-services-parsedmarc-basic-usage} +A very minimal setup which reads incoming reports from an external +email address and saves them to a local Elasticsearch instance looks +like this: + +```nix +services.parsedmarc = { + enable = true; + settings.imap = { + host = "imap.example.com"; + user = "alice@example.com"; + password = "/path/to/imap_password_file"; + watch = true; + }; + provision.geoIp = false; # Not recommended! +}; +``` + +Note that GeoIP provisioning is disabled in the example for +simplicity, but should be turned on for fully functional reports. + +## Local mail +Instead of watching an external inbox, a local inbox can be +automatically provisioned. The recipient's name is by default set to +`dmarc`, but can be configured in +[services.parsedmarc.provision.localMail.recipientName](options.html#opt-services.parsedmarc.provision.localMail.recipientName). You +need to add an MX record pointing to the host. More concretely: for +the example to work, an MX record needs to be set up for +`monitoring.example.com` and the complete email address that should be +configured in the domain's dmarc policy is +`dmarc@monitoring.example.com`. + +```nix +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = monitoring.example.com; + }; + geoIp = false; # Not recommended! + }; +}; +``` + +## Grafana and GeoIP +The reports can be visualized and summarized with parsedmarc's +official Grafana dashboard. For all views to work, and for the data to +be complete, GeoIP databases are also required. The following example +shows a basic deployment where the provisioned Elasticsearch instance +is automatically added as a Grafana datasource, and the dashboard is +added to Grafana as well. + +```nix +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = url; + }; + grafana = { + datasource = true; + dashboard = true; + }; + }; +}; + +# Not required, but recommended for full functionality +services.geoipupdate = { + settings = { + AccountID = 000000; + LicenseKey = "/path/to/license_key_file"; + }; +}; + +services.grafana = { + enable = true; + addr = "0.0.0.0"; + domain = url; + rootUrl = "https://" + url; + protocol = "socket"; + security = { + adminUser = "admin"; + adminPasswordFile = "/path/to/admin_password_file"; + secretKeyFile = "/path/to/secret_key_file"; + }; +}; + +services.nginx = { + enable = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + upstreams.grafana.servers."unix:/${config.services.grafana.socket}" = {}; + virtualHosts.${url} = { + root = config.services.grafana.staticRootPath; + enableACME = true; + forceSSL = true; + locations."/".tryFiles = "$uri @grafana"; + locations."@grafana".proxyPass = "http://grafana"; + }; +}; +users.users.nginx.extraGroups = [ "grafana" ]; +``` diff --git a/nixos/modules/services/monitoring/parsedmarc.nix b/nixos/modules/services/monitoring/parsedmarc.nix index 58137a74d395..e6a72dea0260 100644 --- a/nixos/modules/services/monitoring/parsedmarc.nix +++ b/nixos/modules/services/monitoring/parsedmarc.nix @@ -529,4 +529,9 @@ in description = "DMARC mail recipient"; }; }; + + # Don't edit the docbook xml directly, edit the md and generate it: + # `pandoc parsedmarc.md -t docbook --top-level-division=chapter --extract-media=media -f markdown+smart > parsedmarc.xml` + meta.doc = ./parsedmarc.xml; + meta.maintainers = [ lib.maintainers.talyz ]; } diff --git a/nixos/modules/services/monitoring/parsedmarc.xml b/nixos/modules/services/monitoring/parsedmarc.xml new file mode 100644 index 000000000000..7167b52d0357 --- /dev/null +++ b/nixos/modules/services/monitoring/parsedmarc.xml @@ -0,0 +1,125 @@ + + parsedmarc + + parsedmarc + is a service which parses incoming + DMARC reports and + stores or sends them to a downstream service for further analysis. + In combination with Elasticsearch, Grafana and the included Grafana + dashboard, it provides a handy overview of DMARC reports over time. + +
+ Basic usage + + A very minimal setup which reads incoming reports from an external + email address and saves them to a local Elasticsearch instance + looks like this: + + +services.parsedmarc = { + enable = true; + settings.imap = { + host = "imap.example.com"; + user = "alice@example.com"; + password = "/path/to/imap_password_file"; + watch = true; + }; + provision.geoIp = false; # Not recommended! +}; + + + Note that GeoIP provisioning is disabled in the example for + simplicity, but should be turned on for fully functional reports. + +
+
+ Local mail + + Instead of watching an external inbox, a local inbox can be + automatically provisioned. The recipient’s name is by default set + to dmarc, but can be configured in + services.parsedmarc.provision.localMail.recipientName. + You need to add an MX record pointing to the host. More + concretely: for the example to work, an MX record needs to be set + up for monitoring.example.com and the complete + email address that should be configured in the domain’s dmarc + policy is dmarc@monitoring.example.com. + + +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = monitoring.example.com; + }; + geoIp = false; # Not recommended! + }; +}; + +
+
+ Grafana and GeoIP + + The reports can be visualized and summarized with parsedmarc’s + official Grafana dashboard. For all views to work, and for the + data to be complete, GeoIP databases are also required. The + following example shows a basic deployment where the provisioned + Elasticsearch instance is automatically added as a Grafana + datasource, and the dashboard is added to Grafana as well. + + +services.parsedmarc = { + enable = true; + provision = { + localMail = { + enable = true; + hostname = url; + }; + grafana = { + datasource = true; + dashboard = true; + }; + }; +}; + +# Not required, but recommended for full functionality +services.geoipupdate = { + settings = { + AccountID = 000000; + LicenseKey = "/path/to/license_key_file"; + }; +}; + +services.grafana = { + enable = true; + addr = "0.0.0.0"; + domain = url; + rootUrl = "https://" + url; + protocol = "socket"; + security = { + adminUser = "admin"; + adminPasswordFile = "/path/to/admin_password_file"; + secretKeyFile = "/path/to/secret_key_file"; + }; +}; + +services.nginx = { + enable = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + upstreams.grafana.servers."unix:/${config.services.grafana.socket}" = {}; + virtualHosts.${url} = { + root = config.services.grafana.staticRootPath; + enableACME = true; + forceSSL = true; + locations."/".tryFiles = "$uri @grafana"; + locations."@grafana".proxyPass = "http://grafana"; + }; +}; +users.users.nginx.extraGroups = [ "grafana" ]; + +
+