Merge pull request #60956 from strager/duplicity-tests

duplicity: enable tests

pkgs/tools/backup/duplicity/default.nix

@@ -1,4 +1,8 @@
-{ stdenv, fetchurl, python2Packages, librsync, ncftp, gnupg, rsync, makeWrapper }:
+{ stdenv, fetchpatch, fetchurl, python2Packages, librsync, ncftp, gnupg
+, gnutar
+, par2cmdline
+, utillinux
+, rsync, makeWrapper }:
 
 python2Packages.buildPythonApplication rec {
   name = "duplicity-${version}";
@@ -8,16 +12,46 @@ python2Packages.buildPythonApplication rec {
     url = "https://code.launchpad.net/duplicity/${stdenv.lib.versions.majorMinor version}-series/${version}/+download/${name}.tar.gz";
     sha256 = "0ag9dknslxlasslwfjhqgcqbkb1mvzzx93ry7lch2lfzcdd91am6";
   };
+  patches = [
+    ./gnutar-in-test.patch
+    ./use-installed-scripts-in-test.patch
+
+    # The following patches improve the performance of installCheckPhase:
+    # Ensure all duplicity output is captured in tests
+    (fetchpatch {
+      extraPrefix = "";
+      sha256 = "07ay3mmnw8p2j3v8yvcpjsx0rf2jqly9ablwjpmry23dz9f0mxsd";
+      url = "https://bazaar.launchpad.net/~duplicity-team/duplicity/0.8-series/diff/1359.2.1";
+    })
+    # Minimize time spent sleeping between backups
+    (fetchpatch {
+      extraPrefix = "";
+      sha256 = "0v99q6mvikb8sf68gh3s0zg12pq8fijs87fv1qrvdnc8zvs4pmfs";
+      url = "https://bazaar.launchpad.net/~duplicity-team/duplicity/0.8-series/diff/1359.2.2";
+    })
+    # Remove unnecessary sleeping after running backups in tests
+    (fetchpatch {
+      extraPrefix = "";
+      sha256 = "1bmgp4ilq2gwz2k73fxrqplf866hj57lbyabaqpkvwxhr0ch1jiq";
+      url = "https://bazaar.launchpad.net/~duplicity-team/duplicity/0.8-series/diff/1359.2.3";
+    })
+  ] ++ stdenv.lib.optionals stdenv.isLinux [
+    ./linux-disable-timezone-test.patch
+  ];
 
   buildInputs = [ librsync makeWrapper python2Packages.wrapPython ];
   propagatedBuildInputs = with python2Packages; [
     boto cffi cryptography ecdsa enum idna pygobject3 fasteners
     ipaddress lockfile paramiko pyasn1 pycrypto six
   ];
-  checkInputs = with python2Packages; [ lockfile mock pexpect ];
-
-  # lots of tests are failing, although we get a little further now with the bits in preCheck
-  doCheck = false;
+  checkInputs = [
+    gnupg  # Add 'gpg' to PATH.
+    gnutar  # Add 'tar' to PATH.
+    librsync  # Add 'rdiff' to PATH.
+    par2cmdline  # Add 'par2' to PATH.
+  ] ++ stdenv.lib.optionals stdenv.isLinux [
+    utillinux  # Add 'setsid' to PATH.
+  ] ++ (with python2Packages; [ lockfile mock pexpect ]);
 
   postInstall = ''
     wrapProgram $out/bin/duplicity \
@@ -27,12 +61,30 @@ python2Packages.buildPythonApplication rec {
   '';
 
   preCheck = ''
-    patchShebangs testing
+    wrapPythonProgramsIn "$PWD/testing/overrides/bin" "$pythonPath"
 
-    substituteInPlace testing/__init__.py \
-      --replace 'mkdir testfiles' 'mkdir -p testfiles'
+    # Add 'duplicity' to PATH for tests.
+    # Normally, 'setup.py test' adds 'build/scripts-2.7/' to PATH before running
+    # tests. However, 'build/scripts-2.7/duplicity' is not wrapped, so its
+    # shebang is incorrect and it fails to run inside Nix' sandbox.
+    # In combination with use-installed-scripts-in-test.patch, make 'setup.py
+    # test' use the installed 'duplicity' instead.
+    PATH="$out/bin:$PATH"
+
+    # Don't run developer-only checks (pep8, etc.).
+    export RUN_CODE_TESTS=0
+  '' + stdenv.lib.optionalString stdenv.isDarwin ''
+    # Work around the following error when running tests:
+    # > Max open files of 256 is too low, should be >= 1024.
+    # > Use 'ulimit -n 1024' or higher to correct.
+    ulimit -n 1024
   '';
 
+  # TODO: Fix test failures on macOS 10.13:
+  #
+  # > OSError: out of pty devices
+  doCheck = !stdenv.isDarwin;
+
   meta = with stdenv.lib; {
     description = "Encrypted bandwidth-efficient backup using the rsync algorithm";
     homepage = https://www.nongnu.org/duplicity;

pkgs/tools/backup/duplicity/gnutar-in-test.patch

@@ -0,0 +1,18 @@
+--- a/testing/functional/test_restart.py
++++ b/testing/functional/test_restart.py
+@@ -323,14 +323,7 @@ class RestartTestWithoutEncryption(RestartTest):
+         https://launchpad.net/bugs/929067
+         """
+
+-        if platform.system().startswith('Linux'):
+-            tarcmd = "tar"
+-        elif platform.system().startswith('Darwin'):
+-            tarcmd = "gtar"
+-        elif platform.system().endswith('BSD'):
+-            tarcmd = "gtar"
+-        else:
+-            raise Exception("Platform %s not supported by tar/gtar." % platform.platform())
++        tarcmd = "tar"
+
+         # Intial normal backup
+         self.backup("full", "testfiles/blocktartest")

pkgs/tools/backup/duplicity/linux-disable-timezone-test.patch

@@ -0,0 +1,10 @@
+--- a/testing/unit/test_statistics.py
++++ b/testing/unit/test_statistics.py
+@@ -59,6 +59,7 @@ class StatsObjTest(UnitTestCase):
+         s1 = StatsDeltaProcess()
+         assert s1.get_stat('SourceFiles') == 0
+
++    @unittest.skip("Broken on Linux in Nix' build environment")
+     def test_get_stats_string(self):
+         """Test conversion of stat object into string"""
+         s = StatsObj()

pkgs/tools/backup/duplicity/use-installed-scripts-in-test.patch

@@ -0,0 +1,13 @@
+--- a/setup.py
++++ b/setup.py
+@@ -92,10 +92,6 @@ class TestCommand(test):
+                 except Exception:
+                     pass
+
+-        os.environ['PATH'] = "%s:%s" % (
+-            os.path.abspath(build_scripts_cmd.build_dir),
+-            os.environ.get('PATH'))
+-
+         test.run(self)
+
+     def run_tests(self):