diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index af40e5cc68bd..ab81f9eaabfa 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -4747,6 +4747,12 @@ githubId = 1102396; name = "Jussi Maki"; }; + joaquinito2051 = { + email = "joaquinito2051@gmail.com"; + github = "heroku-miraheze"; + githubId = 61781343; + name = "JoaquĆ­n Rufo Gutierrez"; + }; jobojeha = { email = "jobojeha@jeppener.de"; github = "jobojeha"; diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 3a8289923201..88228c799870 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -977,6 +977,7 @@ ./services/web-servers/shellinabox.nix ./services/web-servers/tomcat.nix ./services/web-servers/traefik.nix + ./services/web-servers/trafficserver.nix ./services/web-servers/ttyd.nix ./services/web-servers/uwsgi.nix ./services/web-servers/varnish/default.nix diff --git a/nixos/modules/services/web-servers/trafficserver.nix b/nixos/modules/services/web-servers/trafficserver.nix new file mode 100644 index 000000000000..db0e2ac0bd05 --- /dev/null +++ b/nixos/modules/services/web-servers/trafficserver.nix @@ -0,0 +1,318 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.trafficserver; + user = config.users.users.trafficserver.name; + group = config.users.groups.trafficserver.name; + + getManualUrl = name: "https://docs.trafficserver.apache.org/en/latest/admin-guide/files/${name}.en.html"; + getConfPath = name: "${pkgs.trafficserver}/etc/trafficserver/${name}"; + + yaml = pkgs.formats.yaml { }; + + fromYAML = f: + let + jsonFile = pkgs.runCommand "in.json" + { + nativeBuildInputs = [ pkgs.remarshal ]; + } '' + yaml2json < "${f}" > "$out" + ''; + in + builtins.fromJSON (builtins.readFile jsonFile); + + mkYamlConf = name: cfg: + if cfg != null then { + "trafficserver/${name}.yaml".source = yaml.generate "${name}.yaml" cfg; + } else { + "trafficserver/${name}.yaml".text = ""; + }; + + mkRecordLines = path: value: + if isAttrs value then + lib.mapAttrsToList (n: v: mkRecordLines (path ++ [ n ]) v) value + else if isInt value then + "CONFIG ${concatStringsSep "." path} INT ${toString value}" + else if isFloat value then + "CONFIG ${concatStringsSep "." path} FLOAT ${toString value}" + else + "CONFIG ${concatStringsSep "." path} STRING ${toString value}"; + + mkRecordsConfig = cfg: concatStringsSep "\n" (flatten (mkRecordLines [ ] cfg)); + mkPluginConfig = cfg: concatStringsSep "\n" (map (p: "${p.path} ${p.arg}") cfg); +in +{ + options.services.trafficserver = { + enable = mkEnableOption "Apache Traffic Server"; + + cache = mkOption { + type = types.lines; + default = ""; + example = "dest_domain=example.com suffix=js action=never-cache"; + description = '' + Caching rules that overrule the origin's caching policy. + + Consult the upstream + documentation for more details. + ''; + }; + + hosting = mkOption { + type = types.lines; + default = ""; + example = "domain=example.com volume=1"; + description = '' + Partition the cache according to origin server or domain + + Consult the + upstream documentation for more details. + ''; + }; + + ipAllow = mkOption { + type = types.nullOr yaml.type; + default = fromYAML (getConfPath "ip_allow.yaml"); + defaultText = "upstream defaults"; + example = literalExample { + ip_allow = [{ + apply = "in"; + ip_addrs = "127.0.0.1"; + action = "allow"; + methods = "ALL"; + }]; + }; + description = '' + Control client access to Traffic Server and Traffic Server connections + to upstream servers. + + Consult the upstream + documentation for more details. + ''; + }; + + logging = mkOption { + type = types.nullOr yaml.type; + default = fromYAML (getConfPath "logging.yaml"); + defaultText = "upstream defaults"; + example = literalExample { }; + description = '' + Configure logs. + + Consult the upstream + documentation for more details. + ''; + }; + + parent = mkOption { + type = types.lines; + default = ""; + example = '' + dest_domain=. method=get parent="p1.example:8080; p2.example:8080" round_robin=true + ''; + description = '' + Identify the parent proxies used in an cache hierarchy. + + Consult the upstream + documentation for more details. + ''; + }; + + plugins = mkOption { + default = [ ]; + + description = '' + Controls run-time loadable plugins available to Traffic Server, as + well as their configuration. + + Consult the upstream + documentation for more details. + ''; + + type = with types; + listOf (submodule { + options.path = mkOption { + type = str; + example = "xdebug.so"; + description = '' + Path to plugin. The path can either be absolute, or relative to + the plugin directory. + ''; + }; + options.arg = mkOption { + type = str; + default = ""; + example = "--header=ATS-My-Debug"; + description = "arguments to pass to the plugin"; + }; + }); + }; + + records = mkOption { + type = with types; + let valueType = (attrsOf (oneOf [ int float str valueType ])) // { + description = "Traffic Server records value"; + }; + in + valueType; + default = { }; + example = literalExample { proxy.config.proxy_name = "my_server"; }; + description = '' + List of configurable variables used by Traffic Server. + + Consult the + upstream documentation for more details. + ''; + }; + + remap = mkOption { + type = types.lines; + default = ""; + example = "map http://from.example http://origin.example"; + description = '' + URL remapping rules used by Traffic Server. + + Consult the + upstream documentation for more details. + ''; + }; + + splitDns = mkOption { + type = types.lines; + default = ""; + example = '' + dest_domain=internal.corp.example named="255.255.255.255:212 255.255.255.254" def_domain=corp.example search_list="corp.example corp1.example" + dest_domain=!internal.corp.example named=255.255.255.253 + ''; + description = '' + Specify the DNS server that Traffic Server should use under specific + conditions. + + Consult the + upstream documentation for more details. + ''; + }; + + sslMulticert = mkOption { + type = types.lines; + default = ""; + example = "dest_ip=* ssl_cert_name=default.pem"; + description = '' + Configure SSL server certificates to terminate the SSL sessions. + + Consult the + upstream documentation for more details. + ''; + }; + + sni = mkOption { + type = types.nullOr yaml.type; + default = null; + example = literalExample { + sni = [{ + fqdn = "no-http2.example.com"; + https = "off"; + }]; + }; + description = '' + Configure aspects of TLS connection handling for both inbound and + outbound connections. + + Consult the upstream + documentation for more details. + ''; + }; + + storage = mkOption { + type = types.lines; + default = "/var/cache/trafficserver 256M"; + example = "/dev/disk/by-id/XXXXX volume=1"; + description = '' + List all the storage that make up the Traffic Server cache. + + Consult the + upstream documentation for more details. + ''; + }; + + strategies = mkOption { + type = types.nullOr yaml.type; + default = null; + description = '' + Specify the next hop proxies used in an cache hierarchy and the + algorithms used to select the next proxy. + + Consult the + upstream documentation for more details. + ''; + }; + + volume = mkOption { + type = types.nullOr yaml.type; + default = ""; + example = "volume=1 scheme=http size=20%"; + description = '' + Manage cache space more efficiently and restrict disk usage by + creating cache volumes of different sizes. + + Consult the + upstream documentation for more details. + ''; + }; + }; + + config = mkIf cfg.enable { + environment.etc = { + "trafficserver/cache.config".text = cfg.cache; + "trafficserver/hosting.config".text = cfg.hosting; + "trafficserver/parent.config".text = cfg.parent; + "trafficserver/plugin.config".text = mkPluginConfig cfg.plugins; + "trafficserver/records.config".text = mkRecordsConfig cfg.records; + "trafficserver/remap.config".text = cfg.remap; + "trafficserver/splitdns.config".text = cfg.splitDns; + "trafficserver/ssl_multicert.config".text = cfg.sslMulticert; + "trafficserver/storage.config".text = cfg.storage; + "trafficserver/volume.config".text = cfg.volume; + } // (mkYamlConf "ip_allow" cfg.ipAllow) + // (mkYamlConf "logging" cfg.logging) + // (mkYamlConf "sni" cfg.sni) + // (mkYamlConf "strategies" cfg.strategies); + + environment.systemPackages = [ pkgs.trafficserver ]; + systemd.packages = [ pkgs.trafficserver ]; + + # Traffic Server does privilege handling independently of systemd, and + # therefore should be started as root + systemd.services.trafficserver = { + enable = true; + wantedBy = [ "multi-user.target" ]; + }; + + # These directories can't be created by systemd because: + # + # 1. Traffic Servers starts as root and switches to an unprivileged user + # afterwards. The runtime directories defined below are assumed to be + # owned by that user. + # 2. The bin/trafficserver script assumes these directories exist. + systemd.tmpfiles.rules = [ + "d '/run/trafficserver' - ${user} ${group} - -" + "d '/var/cache/trafficserver' - ${user} ${group} - -" + "d '/var/lib/trafficserver' - ${user} ${group} - -" + "d '/var/log/trafficserver' - ${user} ${group} - -" + ]; + + services.trafficserver = { + records.proxy.config.admin.user_id = user; + records.proxy.config.body_factory.template_sets_dir = + "${pkgs.trafficserver}/etc/trafficserver/body_factory"; + }; + + users.users.trafficserver = { + description = "Apache Traffic Server"; + isSystemUser = true; + inherit group; + }; + users.groups.trafficserver = { }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 2347673cef1b..4965cd9228d1 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -413,6 +413,7 @@ in # traefik test relies on docker-containers trac = handleTest ./trac.nix {}; traefik = handleTestOn ["x86_64-linux"] ./traefik.nix {}; + trafficserver = handleTest ./trafficserver.nix {}; transmission = handleTest ./transmission.nix {}; trezord = handleTest ./trezord.nix {}; trickster = handleTest ./trickster.nix {}; diff --git a/nixos/tests/trafficserver.nix b/nixos/tests/trafficserver.nix new file mode 100644 index 000000000000..3979a1b4a482 --- /dev/null +++ b/nixos/tests/trafficserver.nix @@ -0,0 +1,176 @@ +# verifies: +# 1. Traffic Server is able to start +# 2. Traffic Server spawns traffic_crashlog upon startup +# 3. Traffic Server proxies HTTP requests according to URL remapping rules +# in 'services.trafficserver.remap' +# 4. Traffic Server applies per-map settings specified with the conf_remap +# plugin +# 5. Traffic Server caches HTTP responses +# 6. Traffic Server processes HTTP PUSH requests +# 7. Traffic Server can load the healthchecks plugin +# 8. Traffic Server logs HTTP traffic as configured +# +# uses: +# - bin/traffic_manager +# - bin/traffic_server +# - bin/traffic_crashlog +# - bin/traffic_cache_tool +# - bin/traffic_ctl +# - bin/traffic_logcat +# - bin/traffic_logstats +# - bin/tspush +import ./make-test-python.nix ({ pkgs, ... }: { + name = "trafficserver"; + meta = with pkgs.lib.maintainers; { + maintainers = [ midchildan ]; + }; + + nodes = { + ats = { pkgs, lib, config, ... }: let + user = config.users.users.trafficserver.name; + group = config.users.groups.trafficserver.name; + healthchecks = pkgs.writeText "healthchecks.conf" '' + /status /tmp/ats.status text/plain 200 500 + ''; + in { + services.trafficserver.enable = true; + + services.trafficserver.records = { + proxy.config.http.server_ports = "80 80:ipv6"; + proxy.config.hostdb.host_file.path = "/etc/hosts"; + proxy.config.log.max_space_mb_headroom = 0; + proxy.config.http.push_method_enabled = 1; + + # check that cache storage is usable before accepting traffic + proxy.config.http.wait_for_cache = 2; + }; + + services.trafficserver.plugins = [ + { path = "healthchecks.so"; arg = toString healthchecks; } + { path = "xdebug.so"; } + ]; + + services.trafficserver.remap = '' + map http://httpbin.test http://httpbin + map http://pristine-host-hdr.test http://httpbin \ + @plugin=conf_remap.so \ + @pparam=proxy.config.url_remap.pristine_host_hdr=1 + map http://ats/tspush http://httpbin/cache \ + @plugin=conf_remap.so \ + @pparam=proxy.config.http.cache.required_headers=0 + ''; + + services.trafficserver.storage = '' + /dev/vdb volume=1 + ''; + + networking.firewall.allowedTCPPorts = [ 80 ]; + virtualisation.emptyDiskImages = [ 256 ]; + services.udev.extraRules = '' + KERNEL=="vdb", OWNER="${user}", GROUP="${group}" + ''; + }; + + httpbin = { pkgs, lib, ... }: let + python = pkgs.python3.withPackages + (ps: with ps; [ httpbin gunicorn gevent ]); + in { + systemd.services.httpbin = { + enable = true; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${python}/bin/gunicorn -b 0.0.0.0:80 httpbin:app -k gevent"; + }; + }; + + networking.firewall.allowedTCPPorts = [ 80 ]; + }; + + client = { pkgs, lib, ... }: { + environment.systemPackages = with pkgs; [ curl ]; + }; + }; + + testScript = { nodes, ... }: let + sampleFile = pkgs.writeText "sample.txt" '' + It's the season of White Album. + ''; + in '' + import json + import re + + ats.wait_for_unit("trafficserver") + ats.wait_for_open_port(80) + httpbin.wait_for_unit("httpbin") + httpbin.wait_for_open_port(80) + + with subtest("Traffic Server is running"): + out = ats.succeed("traffic_ctl server status") + assert out.strip() == "Proxy -- on" + + with subtest("traffic_crashlog is running"): + ats.succeed("pgrep -f traffic_crashlog") + + with subtest("basic remapping works"): + out = client.succeed("curl -vv -H 'Host: httpbin.test' http://ats/headers") + assert json.loads(out)["headers"]["Host"] == "httpbin" + + with subtest("conf_remap plugin works"): + out = client.succeed( + "curl -vv -H 'Host: pristine-host-hdr.test' http://ats/headers" + ) + assert json.loads(out)["headers"]["Host"] == "pristine-host-hdr.test" + + with subtest("caching works"): + out = client.succeed( + "curl -vv -D - -H 'Host: httpbin.test' -H 'X-Debug: X-Cache' http://ats/cache/60 -o /dev/null" + ) + assert "X-Cache: miss" in out + + out = client.succeed( + "curl -vv -D - -H 'Host: httpbin.test' -H 'X-Debug: X-Cache' http://ats/cache/60 -o /dev/null" + ) + assert "X-Cache: hit-fresh" in out + + with subtest("pushing to cache works"): + url = "http://ats/tspush" + + ats.succeed(f"echo {url} > /tmp/urls.txt") + out = ats.succeed( + f"tspush -f '${sampleFile}' -u {url}" + ) + assert "HTTP/1.0 201 Created" in out, "cache push failed" + + out = ats.succeed( + "traffic_cache_tool --spans /etc/trafficserver/storage.config find --input /tmp/urls.txt" + ) + assert "Span: /dev/vdb" in out, "cache not stored on disk" + + out = client.succeed(f"curl {url}").strip() + expected = ( + open("${sampleFile}").read().strip() + ) + assert out == expected, "cache content mismatch" + + with subtest("healthcheck plugin works"): + out = client.succeed("curl -vv http://ats/status -o /dev/null -w '%{http_code}'") + assert out.strip() == "500" + + ats.succeed("touch /tmp/ats.status") + + out = client.succeed("curl -vv http://ats/status -o /dev/null -w '%{http_code}'") + assert out.strip() == "200" + + with subtest("logging works"): + access_log_path = "/var/log/trafficserver/squid.blog" + ats.wait_for_file(access_log_path) + + out = ats.succeed(f"traffic_logcat {access_log_path}").split("\n")[0] + expected = "^\S+ \S+ \S+ TCP_MISS/200 \S+ GET http://httpbin/headers - DIRECT/httpbin application/json$" + assert re.fullmatch(expected, out) is not None, "no matching logs" + + out = json.loads(ats.succeed(f"traffic_logstats -jf {access_log_path}")) + assert out["total"]["error.total"]["req"] == "0", "unexpected log stat" + ''; +}) diff --git a/pkgs/development/libraries/cjose/default.nix b/pkgs/development/libraries/cjose/default.nix new file mode 100644 index 000000000000..57b5c6c1b8bf --- /dev/null +++ b/pkgs/development/libraries/cjose/default.nix @@ -0,0 +1,40 @@ +{ lib +, stdenv +, fetchFromGitHub +, autoreconfHook +, pkg-config +, doxygen +, check +, jansson +, openssl +}: + +stdenv.mkDerivation rec { + pname = "cjose"; + version = "0.6.1"; + + src = fetchFromGitHub { + owner = "cisco"; + repo = "cjose"; + rev = version; + sha256 = "1msyjwmylb5c7jc16ryx3xb9cdwx682ihsm0ni766y6dfwx8bkhp"; + }; + + nativeBuildInputs = [ autoreconfHook pkg-config doxygen ]; + buildInputs = [ jansson openssl ]; + checkInputs = [ check ]; + + configureFlags = [ + "--with-jansson=${jansson}" + "--with-openssl=${openssl.dev}" + ]; + + meta = with lib; { + homepage = "https://github.com/cisco/cjose"; + changelog = "https://github.com/cisco/cjose/blob/${version}/CHANGELOG.md"; + description = "C library for Javascript Object Signing and Encryption"; + license = licenses.mit; + maintainers = with maintainers; [ midchildan ]; + platforms = platforms.all; + }; +} diff --git a/pkgs/servers/http/trafficserver/default.nix b/pkgs/servers/http/trafficserver/default.nix new file mode 100644 index 000000000000..05eb9a17c8e8 --- /dev/null +++ b/pkgs/servers/http/trafficserver/default.nix @@ -0,0 +1,207 @@ +{ lib +, stdenv +, fetchurl +, fetchpatch +, makeWrapper +, nixosTests +, pkg-config +, file +, linuxHeaders +, openssl +, pcre +, perlPackages +, python3 +, xz +, zlib +# recommended dependencies +, withHwloc ? true +, hwloc +, withCurl ? true +, curl +, withCurses ? true +, ncurses +, withCap ? stdenv.isLinux +, libcap +, withUnwind ? stdenv.isLinux +, libunwind +# optional dependencies +, withBrotli ? false +, brotli +, withCjose ? false +, cjose +, withGeoIP ? false +, geoip +, withHiredis ? false +, hiredis +, withImageMagick ? false +, imagemagick +, withJansson ? false +, jansson +, withKyotoCabinet ? false +, kyotocabinet +, withLuaJIT ? false +, luajit +, withMaxmindDB ? false +, libmaxminddb +# optional features +, enableWCCP ? false +}: + +stdenv.mkDerivation rec { + pname = "trafficserver"; + version = "9.0.1"; + + src = fetchurl { + url = "mirror://apache/trafficserver/trafficserver-${version}.tar.bz2"; + sha256 = "1q164pvfmbqh3gzy3bqy96lwd0fdbhz78r06pd92p7rmkqwx005z"; + }; + + patches = [ + # Adds support for NixOS + # https://github.com/apache/trafficserver/pull/7697 + (fetchpatch { + url = "https://github.com/apache/trafficserver/commit/19d3af481cf74c91fbf713fc9d2f8b138ed5fbaf.diff"; + sha256 = "0z1ikgpp00rzrrcqh97931586yn9wbksgai9xlkcjd5cg8gq0150"; + }) + + # Fixes a bug in tspush which pushes incorrect contents to cache + # https://github.com/apache/trafficserver/pull/7696 + (fetchpatch { + url = "https://github.com/apache/trafficserver/commit/b08215272872f452787915cd3a8e0b0ea0b88385.diff"; + sha256 = "0axk8x1xvd8wvpgcxgyqqg7kgxyxwfgwmisq3xnk1da0cqv9cx9f"; + }) + ]; + + # NOTE: The upstream README indicates that flex is needed for some features, + # but it actually seems to be unnecessary as of this commit[1]. The detection + # logic for bison and flex is still present in the build script[2], but no + # other code seems to depend on it. This situation is susceptible to change + # though, so it's a good idea to inspect the build scripts periodically. + # + # [1]: https://github.com/apache/trafficserver/pull/5617 + # [2]: https://github.com/apache/trafficserver/blob/3fd2c60/configure.ac#L742-L788 + nativeBuildInputs = [ makeWrapper pkg-config file python3 ] + ++ (with perlPackages; [ perl ExtUtilsMakeMaker ]) + ++ lib.optionals stdenv.isLinux [ linuxHeaders ]; + + buildInputs = [ + openssl + pcre + perlPackages.perl + ] ++ lib.optional withBrotli brotli + ++ lib.optional withCap libcap + ++ lib.optional withCjose cjose + ++ lib.optional withCurl curl + ++ lib.optional withGeoIP geoip + ++ lib.optional withHiredis hiredis + ++ lib.optional withHwloc hwloc + ++ lib.optional withImageMagick imagemagick + ++ lib.optional withJansson jansson + ++ lib.optional withKyotoCabinet kyotocabinet + ++ lib.optional withCurses ncurses + ++ lib.optional withLuaJIT luajit + ++ lib.optional withUnwind libunwind + ++ lib.optional withMaxmindDB libmaxminddb; + + outputs = [ "out" "man" ]; + + postPatch = '' + patchShebangs \ + iocore/aio/test_AIO.sample \ + src/traffic_via/test_traffic_via \ + src/traffic_logstats/tests \ + tools/check-unused-dependencies + + substituteInPlace configure --replace '/usr/bin/file' '${file}/bin/file' + '' + lib.optionalString stdenv.isLinux '' + substituteInPlace configure \ + --replace '/usr/include/linux' '${linuxHeaders}/include/linux' + '' + lib.optionalString stdenv.isDarwin '' + # 'xcrun leaks' probably requires non-free XCode + substituteInPlace iocore/net/test_certlookup.cc \ + --replace 'xcrun leaks' 'true' + ''; + + configureFlags = [ + "--enable-layout=NixOS" + "--enable-experimental-plugins" + (lib.enableFeature enableWCCP "wccp") + + # the configure script can't auto-locate the following from buildInputs + "--with-lzma=${xz.dev}" + "--with-zlib=${zlib.dev}" + (lib.withFeatureAs withHiredis "hiredis" hiredis) + ]; + + installFlags = [ + "pkgsysconfdir=${placeholder "out"}/etc/trafficserver" + + # replace runtime directories with an install-time placeholder directory + "pkgcachedir=${placeholder "out"}/.install-trafficserver" + "pkglocalstatedir=${placeholder "out"}/.install-trafficserver" + "pkglogdir=${placeholder "out"}/.install-trafficserver" + "pkgruntimedir=${placeholder "out"}/.install-trafficserver" + ]; + + postInstall = '' + substituteInPlace rc/trafficserver.service --replace "syslog.target" "" + install -Dm644 rc/trafficserver.service $out/lib/systemd/system/trafficserver.service + + wrapProgram $out/bin/tspush \ + --set PERL5LIB '${with perlPackages; makePerlPath [ URI ]}' \ + --prefix PATH : "${lib.makeBinPath [ file ]}" + + find "$out" -name '*.la' -delete + + # ensure no files actually exist in this directory + rmdir $out/.install-trafficserver + ''; + + installCheckPhase = let + expected = '' + Via header is [uScMsEf p eC:t cCMp sF], Length is 22 + Via Header Details: + Request headers received from client :simple request (not conditional) + Result of Traffic Server cache lookup for URL :miss (a cache "MISS") + Response information received from origin server :error in response + Result of document write-to-cache: :no cache write performed + Proxy operation result :unknown + Error codes (if any) :connection to server failed + Tunnel info :no tunneling + Cache Type :cache + Cache Lookup Result :cache miss (url not in cache) + Parent proxy connection status :no parent proxy or unknown + Origin server connection status :connection open failed + ''; + in '' + runHook preInstallCheck + diff -Naur <($out/bin/traffic_via '[uScMsEf p eC:t cCMp sF]') - <