Merge pull request #22297 from nand0p/buildbot-0.9.3

buildbot: 0.9.0.post1 -> 0.9.3

nixos/modules/module-list.nix

@@ -141,6 +141,7 @@
   ./services/computing/torque/mom.nix
   ./services/computing/slurm/slurm.nix
   ./services/continuous-integration/buildbot/master.nix
+  ./services/continuous-integration/buildbot/worker.nix
   ./services/continuous-integration/buildkite-agent.nix
   ./services/continuous-integration/hydra/default.nix
   ./services/continuous-integration/gitlab-runner.nix

nixos/modules/services/continuous-integration/buildbot/master.nix

@@ -7,7 +7,7 @@ with lib;
 let
   cfg = config.services.buildbot-master;
   escapeStr = s: escape ["'"] s;
-  masterCfg = pkgs.writeText "master.cfg" ''
+  masterCfg = if cfg.masterCfg == null then pkgs.writeText "master.cfg" ''
     from buildbot.plugins import *
     factory = util.BuildFactory()
     c = BuildmasterConfig = dict(
@@ -27,9 +27,8 @@ let
       factory.addStep(step)
 
     ${cfg.extraConfig}
-  '';
-
-  configFile = if cfg.masterCfg == null then masterCfg else cfg.masterCfg;
+  ''
+  else pkgs.writeText "master.cfg" cfg.masterCfg;
 
 in {
   options = {
@@ -67,15 +66,13 @@ in {
       };
 
       masterCfg = mkOption {
-        type = with types; nullOr path;
+        type = types.str;
         description = ''
-          Optionally pass path to raw master.cfg file.
+          Optionally pass raw master.cfg file as string.
           Other options in this configuration will be ignored.
         '';
         default = null;
-        example = literalExample ''
-          pkgs.writeText "master.cfg" "BuildmasterConfig = c = {}"
-        '';
+        example = "BuildmasterConfig = c = {}";
       };
 
       schedulers = mkOption {
@@ -99,9 +96,9 @@ in {
         type = types.listOf types.str;
         description = "List of Workers.";
         default = [
-          "worker.Worker('default-worker', 'password')"
+          "worker.Worker('example-worker', 'pass')"
         ];
-        example = [ "worker.LocalWorker('default-worker')" ];
+        example = [ "worker.LocalWorker('example-worker')" ];
       };
 
       status = mkOption {
@@ -209,7 +206,7 @@ in {
 
     users.extraUsers = optional (cfg.user == "buildbot") {
       name = "buildbot";
-      description = "buildbot user";
+      description = "Buildbot User.";
       isNormalUser = true;
       createHome = true;
       home = cfg.home;
@@ -219,7 +216,7 @@ in {
     };
 
     systemd.services.buildbot-master = {
-      description = "Buildbot Continuous Integration Server";
+      description = "Buildbot Continuous Integration Server.";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
       path = cfg.packages;
@@ -233,9 +230,8 @@ in {
       };
 
       preStart = ''
-        mkdir -vp ${cfg.buildbotDir}
-        chown -c ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
-        ln -sf ${configFile} ${cfg.buildbotDir}/master.cfg
+        ${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir}
+        ${pkgs.coreutils}/bin/ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg
         ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir}
       '';
 
@@ -247,4 +243,6 @@ in {
     };
   };
 
+  meta.maintainers = with lib.maintainers; [ nand0p Mic92 ];
+
 }

nixos/modules/services/continuous-integration/buildbot/worker.nix

@@ -0,0 +1,128 @@
+# NixOS module for Buildbot Worker.
+
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.buildbot-worker;
+
+in {
+  options = {
+    services.buildbot-worker = {
+
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to enable the Buildbot Worker.";
+      };
+
+      user = mkOption {
+        default = "bbworker";
+        type = types.str;
+        description = "User the buildbot Worker should execute under.";
+      };
+
+      group = mkOption {
+        default = "bbworker";
+        type = types.str;
+        description = "Primary group of buildbot Worker user.";
+      };
+
+      extraGroups = mkOption {
+        type = types.listOf types.str;
+        default = [ "nixbld" ];
+        description = "List of extra groups that the Buildbot Worker user should be a part of.";
+      };
+
+      home = mkOption {
+        default = "/home/bbworker";
+        type = types.path;
+        description = "Buildbot home directory.";
+      };
+
+      buildbotDir = mkOption {
+        default = "${cfg.home}/worker";
+        type = types.path;
+        description = "Specifies the Buildbot directory.";
+      };
+
+      workerUser = mkOption {
+        default = "example-worker";
+        type = types.str;
+        description = "Specifies the Buildbot Worker user.";
+      };
+
+      workerPass = mkOption {
+        default = "pass";
+        type = types.str;
+        description = "Specifies the Buildbot Worker password.";
+      };
+
+      masterUrl = mkOption {
+        default = "localhost:9989";
+        type = types.str;
+        description = "Specifies the Buildbot Worker connection string.";
+      };
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.buildbot-worker;
+        description = "Package to use for buildbot worker.";
+        example = pkgs.buildbot-worker;
+      };
+
+      packages = mkOption {
+        default = [ ];
+        example = [ pkgs.git ];
+        type = types.listOf types.package;
+        description = "Packages to add to PATH for the buildbot process.";
+      };
+
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.extraGroups = optional (cfg.group == "bbworker") {
+      name = "bbworker";
+    };
+
+    users.extraUsers = optional (cfg.user == "bbworker") {
+      name = "bbworker";
+      description = "Buildbot Worker User.";
+      isNormalUser = true;
+      createHome = true;
+      home = cfg.home;
+      group = cfg.group;
+      extraGroups = cfg.extraGroups;
+      useDefaultShell = true;
+    };
+
+    systemd.services.buildbot-worker = {
+      description = "Buildbot Worker.";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "buildbot-master.service" ];
+      path = cfg.packages;
+
+      preStart = ''
+        # NOTE: ensure master has time to start in case running on localhost
+        ${pkgs.coreutils}/bin/sleep 4
+        ${pkgs.coreutils}/bin/mkdir -vp ${cfg.buildbotDir}
+        ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass}
+      '';
+
+      serviceConfig = {
+        Type = "forking";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.home;
+        ExecStart = "${cfg.package}/bin/buildbot-worker start ${cfg.buildbotDir}";
+      };
+
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ nand0p ];
+
+}

pkgs/development/python-modules/incremental/default.nix

@@ -0,0 +1,19 @@
+{ stdenv, buildPythonPackage, fetchurl }:
+
+buildPythonPackage rec {
+  name = "${pname}-${version}";
+  pname = "incremental";
+  version = "16.10.1";
+
+  src = fetchurl {
+    url = "mirror://pypi/i/${pname}/${name}.tar.gz";
+    sha256 = "0hh382gsj5lfl3fsabblk2djngl4n5yy90xakinasyn41rr6pb8l";
+  };
+
+  meta = with stdenv.lib; {
+    homepage = http://github.com/twisted/treq;
+    description = "Incremental is a small library that versions your Python projects";
+    license = licenses.mit;
+    maintainers = with maintainers; [ nand0p ];
+  };
+}

pkgs/development/python-modules/treq/default.nix

@@ -0,0 +1,51 @@
+{ stdenv, fetchurl, buildPythonPackage, service-identity, requests2,
+  six, mock, twisted, incremental, coreutils, gnumake, pep8, sphinx,
+  openssl, pyopenssl }:
+
+buildPythonPackage rec {
+  name = "${pname}-${version}";
+  pname = "treq";
+  version = "16.12.0";
+
+  src = fetchurl {
+    url = "mirror://pypi/t/${pname}/${name}.tar.gz";
+    sha256 = "1aci3f3rmb5mdf4s6s4k4kghmnyy784cxgi3pz99m5jp274fs25h";
+  };
+
+  buildInputs = [
+    pep8
+    mock
+  ];
+
+  propagatedBuildInputs = [
+    service-identity
+    requests2
+    twisted
+    incremental
+    sphinx
+    six
+    openssl
+    pyopenssl
+  ];
+
+  checkPhase = ''
+    ${pep8}/bin/pep8 --ignore=E902 treq
+    trial treq
+  '';
+
+  doCheck = false;
+  # Failure: twisted.web._newclient.RequestTransmissionFailed: [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'ssl3_get_server_certificate', 'certificate verify failed')]>]
+
+  postBuild = ''
+    ${coreutils}/bin/mkdir -pv treq
+    ${coreutils}/bin/echo "${version}" | ${coreutils}/bin/tee treq/_version
+    cd docs && ${gnumake}/bin/make html && cd ..
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = http://github.com/twisted/treq;
+    description = "A requests-like API built on top of twisted.web's Agent";
+    license = licenses.mit;
+    maintainers = with maintainers; [ nand0p ];
+  };
+}

pkgs/development/tools/build-managers/buildbot/default.nix

@@ -1,21 +1,13 @@
-{ stdenv,
-  lib,
-  pythonPackages,
-  fetchurl,
-  coreutils,
-  openssh,
-  buildbot-worker,
-  plugins ? [],
-  enableLocalWorker ? false
-}:
+{ stdenv, lib, fetchurl, coreutils, openssh, buildbot-worker, makeWrapper,
+  pythonPackages, gnused, plugins ? [] }:
 
 pythonPackages.buildPythonApplication (rec {
   name = "${pname}-${version}";
   pname = "buildbot";
-  version = "0.9.0.post1";
+  version = "0.9.3";
   src = fetchurl {
     url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-    sha256 = "18rnsp691cnmbymlch6czx3mrcmifmf6dk97h9nslgfkkyf25n5g";
+    sha256 = "1yw7knk5dcvwms14vqwlp89flhjf8567l17s9cq7vydh760nmg62";
   };
 
   buildInputs = with pythonPackages; [
@@ -31,7 +23,11 @@ pythonPackages.buildPythonApplication (rec {
     pylint
     astroid
     pyflakes
-  ] ++ lib.optionals (enableLocalWorker) [openssh];
+    openssh
+    buildbot-worker
+    makeWrapper
+    treq
+  ];
 
   propagatedBuildInputs = with pythonPackages; [
 
@@ -39,7 +35,6 @@ pythonPackages.buildPythonApplication (rec {
     twisted
     jinja2
     zope_interface
-    future
     sqlalchemy
     sqlalchemy_migrate
     future
@@ -61,32 +56,21 @@ pythonPackages.buildPythonApplication (rec {
     ramlfications
     sphinx-jinja
 
-  ] ++ plugins ++
-  lib.optionals (enableLocalWorker) [buildbot-worker];
-
-  preInstall = ''
-    # writes out a file that can't be read properly
-    sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py
-  '';
+  ] ++ plugins;
 
   postPatch = ''
-    # re-hardcode path to tail
-    sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
+    ${gnused}/bin/sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
   '';
 
   postFixup = ''
-    mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot
-    echo "#!/bin/sh" > $out/bin/buildbot
-    echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot
-    echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot
-    chmod -c 555 $out/bin/buildbot
+    makeWrapper $out/bin/.buildbot-wrapped $out/bin/buildbot --set PYTHONPATH "$PYTHONPATH"
   '';
 
   meta = with stdenv.lib; {
     homepage = http://buildbot.net/;
     description = "Continuous integration system that automates the build/test cycle";
     maintainers = with maintainers; [ nand0p ryansydnor ];
-    platforms = platforms.all;
+    platforms = platforms.linux;
     license = licenses.gpl2;
   };
 })

pkgs/development/tools/build-managers/buildbot/plugins.nix

@@ -4,11 +4,11 @@ let
   buildbot-pkg = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot-pkg";
-    version = "0.9.0.post1";
+    version = "0.9.3";
 
     src = fetchurl {
       url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-      sha256 = "0frmnc73dsyc9mjnrnpm4vdrwb7c63gc6maq6xvlp486v7sdhjbi";
+      sha256 = "02949cvmghyh313i1hmplwxp3nzq789kk85xjx2ir82cpr1d6h6j";
     };
 
     propagatedBuildInputs = with pythonPackages; [ setuptools ];
@@ -26,14 +26,15 @@ in {
   www = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot_www";
-    version = "0.9.0.post1";
+    version = "0.9.3";
 
     # NOTE: wheel is used due to buildbot circular dependency
     format = "wheel";
 
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/02/d0/fc56ee27a09498638a47dcc5637ee5412ab7a67bfb4b3ff47e041f3d7b66/${name}-py2-none-any.whl";
-      sha256 = "14ghch67k6090736n89l401swz7r9hnk2zlmdb59niq8lg7dyg9q";
+    src = pythonPackages.fetchPypi {
+      inherit pname version format;
+      python = "py2";
+      sha256 = "0yggg6mcykcnv41srl2sp2zwx2r38vb6a8jgxh1a4825mspm2jf7";
     };
 
     meta = with stdenv.lib; {
@@ -48,14 +49,14 @@ in {
   console-view = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot-console-view";
-    version = "0.9.0.post1";
+    version = "0.9.3";
 
     src = fetchurl {
       url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-      sha256 = "0dc7rb7mrpva5gj7l57i96a78d6yj28pkkj9hfim1955z9dgn58l";
+      sha256 = "1rkzakm05x72nvdivc5bc3gab3nyasdfvlwnwril90jj9q1b92dk";
     };
 
-    propagatedBuildInputs = [ buildbot-pkg ];
+    propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
 
     meta = with stdenv.lib; {
       homepage = http://buildbot.net/;
@@ -69,14 +70,14 @@ in {
   waterfall-view = pythonPackages.buildPythonPackage rec {
     name = "${pname}-${version}";
     pname = "buildbot-waterfall-view";
-    version = "0.9.0.post1";
+    version = "0.9.3";
 
     src = fetchurl {
       url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-      sha256 = "0x9vvw15zzgj4w3qcxh8r10rb36ni0qh1215y7wbawh5lggnjm0g";
+      sha256 = "033x2cs0znhk1j0lw067nmjw2m7yy1fdq5qch0sx50jnpjiq6g6g";
     };
 
-    propagatedBuildInputs = [ buildbot-pkg ];
+    propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ];
 
     meta = with stdenv.lib; {
       homepage = http://buildbot.net/;

pkgs/development/tools/build-managers/buildbot/worker.nix

@@ -1,18 +1,22 @@
-{ stdenv, fetchurl, pythonPackages }:
+{ stdenv, fetchurl, gnused, coreutils, pythonPackages }:
 
 pythonPackages.buildPythonApplication (rec {
   name = "${pname}-${version}";
   pname = "buildbot-worker";
-  version = "0.9.0.post1";
+  version = "0.9.3";
 
   src = fetchurl {
     url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-    sha256 = "1f8ij3y62r9z7qv92x21rg9h9whhakkwv59rgniq09j64ggjz8lx";
+    sha256 = "176kp04g4c7gj15f73wppraqrirbfclyx214gcz966019niikcsp";
   };
 
   buildInputs = with pythonPackages; [ setuptoolsTrial mock ];
   propagatedBuildInputs = with pythonPackages; [ twisted future ];
 
+  postPatch = ''
+    ${gnused}/bin/sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot_worker/scripts/logwatcher.py
+  '';
+
   meta = with stdenv.lib; {
     homepage = http://buildbot.net/;
     description = "Buildbot Worker Daemon";

pkgs/top-level/all-packages.nix

@@ -6144,7 +6144,6 @@ with pkgs;
   };
   buildbot-full = self.buildbot.override {
     plugins = with self.buildbot-plugins; [ www console-view waterfall-view ];
-    enableLocalWorker = true;
   };
 
   buildkite-agent = callPackage ../development/tools/continuous-integration/buildkite-agent { };

pkgs/top-level/python-packages.nix

@@ -32131,6 +32131,10 @@ EOF
     };
   };
 
+  incremental = callPackage ../development/python-modules/incremental { };
+
+  treq = callPackage ../development/python-modules/treq { };
+
 });
 
 in fix' (extends overrides packages)