solr: 4.10.3 -> 7.5.0, refactor service to reflect major changes in version bump, NixOS test included

nixos/doc/manual/release-notes/rl-1903.xml

@@ -182,6 +182,13 @@
       options can occour more than once in the configuration.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The <literal>solr</literal> package has been upgraded from 4.10.3 to 7.5.0 and has undergone
+      some major changes. The <literal>services.solr</literal> module has been updated to reflect
+      these changes. Please review http://lucene.apache.org/solr/ carefully before upgrading.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/misc/ids.nix

@@ -333,6 +333,7 @@
       lidarr = 306;
       slurm = 307;
       kapacitor = 308;
+      solr = 309;
 
       # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
 
@@ -626,6 +627,7 @@
       lidarr = 306;
       slurm = 307;
       kapacitor = 308;
+      solr = 309;
 
       # When adding a gid, make sure it doesn't match an existing
       # uid. Users and groups with the same name should have equal

nixos/modules/services/search/solr.nix

@@ -6,142 +6,105 @@ let
 
   cfg = config.services.solr;
 
-  # Assemble all jars needed for solr
-  solrJars = pkgs.stdenv.mkDerivation {
-    name = "solr-jars";
-
-    src = pkgs.fetchurl {
-      url = http://archive.apache.org/dist/tomcat/tomcat-5/v5.5.36/bin/apache-tomcat-5.5.36.tar.gz;
-      sha256 = "01mzvh53wrs1p2ym765jwd00gl6kn8f9k3nhdrnhdqr8dhimfb2p";
-    };
-
-    installPhase = ''
-      mkdir -p $out/lib
-      cp common/lib/*.jar $out/lib/
-      ln -s ${pkgs.ant}/lib/ant/lib/ant.jar $out/lib/
-      ln -s ${cfg.solrPackage}/lib/ext/* $out/lib/
-      ln -s ${pkgs.jdk.home}/lib/tools.jar $out/lib/
-    '' + optionalString (cfg.extraJars != []) ''
-      for f in ${concatStringsSep " " cfg.extraJars}; do
-         cp $f $out/lib
-      done
-    '';
-  };
-
-in {
+in
 
+{
   options = {
     services.solr = {
-      enable = mkOption {
-        type = types.bool;
-        default = false;
-        description = ''
-          Enables the solr service.
-        '';
-      };
+      enable = mkEnableOption "Enables the solr service.";
 
-      javaPackage = mkOption {
-        type = types.package;
-        default = pkgs.jre;
-        defaultText = "pkgs.jre";
-        description = ''
-          Which Java derivation to use for running solr.
-        '';
-      };
-
-      solrPackage = mkOption {
+      package = mkOption {
         type = types.package;
         default = pkgs.solr;
         defaultText = "pkgs.solr";
-        description = ''
-          Which solr derivation to use for running solr.
-        '';
+        description = "Which Solr package to use.";
       };
 
-      extraJars = mkOption {
-        type = types.listOf types.path;
-        default = [];
-        description = ''
-          List of paths pointing to jars. Jars are copied to commonLibFolder to be available to java/solr.
-        '';
+      port = mkOption {
+        type = types.int;
+        default = 8983;
+        description = "Port on which Solr is ran.";
       };
 
-      log4jConfiguration = mkOption {
-        type = types.lines;
-        default = ''
-          log4j.rootLogger=INFO, stdout
-          log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-          log4j.appender.stdout.Target=System.out
-          log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-          log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-        '';
-        description = ''
-          Contents of the <literal>log4j.properties</literal> used. By default,
-          everything is logged to stdout (picked up by systemd) with level INFO.
-        '';
-      };
-
-      user = mkOption {
-        type = types.str;
-        description = ''
-          The user that should run the solr process and.
-          the working directories.
-        '';
-      };
-
-      group = mkOption {
-        type = types.str;
-        description = ''
-          The group that will own the working directory.
-        '';
-      };
-
-      solrHome = mkOption {
-        type = types.str;
-        description = ''
-          The solr home directory. It is your own responsibility to
-          make sure this directory contains a working solr configuration,
-          and is writeable by the the user running the solr service.
-          Failing to do so, the solr will not start properly.
-        '';
+      stateDir = mkOption {
+        type = types.path;
+        default = "/var/lib/solr";
+        description = "The solr home directory containing config, data, and logging files.";
       };
 
       extraJavaOptions = mkOption {
         type = types.listOf types.str;
         default = [];
-        description = ''
-          Extra command line options given to the java process running
-          solr.
-        '';
+        description = "Extra command line options given to the java process running Solr.";
       };
 
-      extraWinstoneOptions = mkOption {
-        type = types.listOf types.str;
-        default = [];
-        description = ''
-          Extra command line options given to the Winstone, which is
-          the servlet container hosting solr.
-        '';
+      user = mkOption {
+        type = types.str;
+        default = "solr";
+        description = "User under which Solr is ran.";
+      };
+
+      group = mkOption {
+        type = types.str;
+        default = "solr";
+        description = "Group under which Solr is ran.";
       };
     };
   };
 
   config = mkIf cfg.enable {
 
-    services.winstone.solr = {
-      serviceName = "solr";
-      inherit (cfg) user group javaPackage;
-      warFile = "${cfg.solrPackage}/lib/solr.war";
-      extraOptions = [
-        "--commonLibFolder=${solrJars}/lib"
-        "--useJasper"
-      ] ++ cfg.extraWinstoneOptions;
-      extraJavaOptions = [
-        "-Dsolr.solr.home=${cfg.solrHome}"
-        "-Dlog4j.configuration=file://${pkgs.writeText "log4j.properties" cfg.log4jConfiguration}"
-      ] ++ cfg.extraJavaOptions;
+    environment.systemPackages = [ cfg.package ];
+
+    systemd.services.solr = {
+      after = [ "network.target" "remote-fs.target" "nss-lookup.target" "systemd-journald-dev-log.socket" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment = {
+        SOLR_HOME = "${cfg.stateDir}/data";
+        LOG4J_PROPS = "${cfg.stateDir}/log4j2.xml";
+        SOLR_LOGS_DIR = "${cfg.stateDir}/logs";
+        SOLR_PORT = "${toString cfg.port}";
+      };
+      path = with pkgs; [
+        gawk
+        procps
+      ];
+      preStart = ''
+        mkdir -p "${cfg.stateDir}/data";
+        mkdir -p "${cfg.stateDir}/logs";
+
+        if ! test -e "${cfg.stateDir}/data/solr.xml"; then
+          install -D -m0640 ${cfg.package}/server/solr/solr.xml "${cfg.stateDir}/data/solr.xml"
+          install -D -m0640 ${cfg.package}/server/solr/zoo.cfg "${cfg.stateDir}/data/zoo.cfg"
+        fi
+
+        if ! test -e "${cfg.stateDir}/log4j2.xml"; then
+          install -D -m0640 ${cfg.package}/server/resources/log4j2.xml "${cfg.stateDir}/log4j2.xml"
+        fi
+      '';
+
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart="${cfg.package}/bin/solr start -f -a \"${concatStringsSep " " cfg.extraJavaOptions}\"";
+        ExecStop="${cfg.package}/bin/solr stop";
+      };
     };
 
+    users.users = optionalAttrs (cfg.user == "solr") (singleton
+      { name = "solr";
+        group = cfg.group;
+        home = cfg.stateDir;
+        createHome = true;
+        uid = config.ids.uids.solr;
+      });
+
+    users.groups = optionalAttrs (cfg.group == "solr") (singleton
+      { name = "solr";
+        gid = config.ids.gids.solr;
+      });
+
   };
 
 }

nixos/release.nix

@@ -410,6 +410,7 @@ in rec {
   tests.slurm = callTest tests/slurm.nix {};
   tests.smokeping = callTest tests/smokeping.nix {};
   tests.snapper = callTest tests/snapper.nix {};
+  tests.solr = callTest tests/solr.nix {};
   #tests.statsd = callTest tests/statsd.nix {}; # statsd is broken: #45946
   tests.strongswan-swanctl = callTest tests/strongswan-swanctl.nix {};
   tests.sudo = callTest tests/sudo.nix {};

nixos/tests/solr.nix

@@ -0,0 +1,47 @@
+import ./make-test.nix ({ pkgs, lib, ... }:
+{
+  name = "solr";
+  meta.maintainers = [ lib.maintainers.aanderse ];
+
+  machine =
+    { config, pkgs, ... }:
+    {
+      # Ensure the virtual machine has enough memory for Solr to avoid the following error:
+      #
+      #   OpenJDK 64-Bit Server VM warning:
+      #     INFO: os::commit_memory(0x00000000e8000000, 402653184, 0)
+      #     failed; error='Cannot allocate memory' (errno=12)
+      #
+      #   There is insufficient memory for the Java Runtime Environment to continue.
+      #   Native memory allocation (mmap) failed to map 402653184 bytes for committing reserved memory.
+      virtualisation.memorySize = 2000;
+
+      services.solr.enable = true;
+    };
+
+  testScript = ''
+    startAll;
+
+    $machine->waitForUnit('solr.service');
+    $machine->waitForOpenPort('8983');
+    $machine->succeed('curl --fail http://localhost:8983/solr/');
+
+    # adapted from pkgs.solr/examples/films/README.txt
+    $machine->succeed('sudo -u solr solr create -c films');
+    $machine->succeed(q(curl http://localhost:8983/solr/films/schema -X POST -H 'Content-type:application/json' --data-binary '{
+      "add-field" : {
+        "name":"name",
+        "type":"text_general",
+        "multiValued":false,
+        "stored":true
+      },
+      "add-field" : {
+        "name":"initial_release_date",
+        "type":"pdate",
+        "stored":true
+      }
+    }')) =~ /"status":0/ or die;
+    $machine->succeed('sudo -u solr post -c films ${pkgs.solr}/example/films/films.json');
+    $machine->succeed('curl http://localhost:8983/solr/films/query?q=name:batman') =~ /"name":"Batman Begins"/ or die;
+  '';
+})

pkgs/servers/search/solr/default.nix

@@ -1,20 +1,27 @@
-{ stdenv, fetchurl }:
+{ stdenv, fetchurl, jre, makeWrapper }:
 
 stdenv.mkDerivation rec {
   name = "solr-${version}";
-  version = "4.10.3";
+  version = "7.5.0";
 
   src = fetchurl {
     url = "mirror://apache/lucene/solr/${version}/solr-${version}.tgz";
-    sha256 = "1dp269jka4q62qhv47j91wsrsnbxfn23lsx6qcycbijrlyh28w5c";
+    sha256 = "1g6f58j2pzb73phj4hfri9mj7vmql72by7w3xrbq1pbnqgzxmhpa";
   };
 
-  phases = [ "unpackPhase" "installPhase" ];
+  nativeBuildInputs = [ makeWrapper ];
 
   installPhase = ''
-    mkdir -p $out/lib
-    cp dist/${name}.war $out/lib/solr.war
-    cp -r example/lib/ext $out/lib/ext
+    mkdir -p $out $out/bin
+
+    cp -r bin/solr bin/post $out/bin/
+    cp -r contrib $out/
+    cp -r dist $out/
+    cp -r example $out/
+    cp -r server $out/
+
+    wrapProgram $out/bin/solr --set JAVA_HOME "${jre}"
+    wrapProgram $out/bin/post --set JAVA_HOME "${jre}"
   '';
 
   meta = with stdenv.lib; {
@@ -22,7 +29,7 @@ stdenv.mkDerivation rec {
     description = "Open source enterprise search platform from the Apache Lucene project";
     license = licenses.asl20;
     platforms = platforms.all;
-    maintainers = [ maintainers.rickynils maintainers.domenkozar ];
+    maintainers = [ maintainers.rickynils maintainers.domenkozar maintainers.aanderse ];
   };
 
 }