Merge pull request #205623 from Artturin/structuredtests1

tests.stdenv.hooks: add more tests

pkgs/build-support/setup-hooks/audit-tmpdir.sh

@@ -16,7 +16,7 @@ auditTmpdir() {
     header "checking for references to $TMPDIR/ in $dir..."
 
     local i
-    while IFS= read -r -d $'\0' i; do
+    find "$dir" -type f -print0 | while IFS= read -r -d $'\0' i; do
         if [[ "$i" =~ .build-id ]]; then continue; fi
 
         if isELF "$i"; then
@@ -35,7 +35,7 @@ auditTmpdir() {
             fi
         fi
 
-    done < <(find "$dir" -type f -print0)
+    done
 
     stopNest
 }

pkgs/build-support/setup-hooks/auto-patchelf.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
 
 declare -a autoPatchelfLibs
 declare -a extraAutoPatchelfLibs

pkgs/build-support/setup-hooks/desktop-to-darwin-bundle.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
 fixupOutputHooks+=('convertDesktopFiles $prefix')
 
 # Get a param out of a desktop file. First parameter is the file and the second

pkgs/build-support/setup-hooks/move-docs.sh

@@ -1,6 +1,4 @@
-# This setup hook moves $out/{man,doc,info} to $out/share; moves
-# $out/share/man to $man/share/man; and moves $out/share/doc to
-# $man/share/doc.
+# This setup hook moves $out/{man,doc,info} to $out/share.
 
 preFixupHooks+=(_moveToShare)
 

pkgs/build-support/setup-hooks/move-systemd-user-units.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
 
 # This setup hook, for each output, moves everything in
 # $output/lib/systemd/user to $output/share/systemd/user, and replaces

pkgs/development/libraries/oniguruma/default.nix

@@ -1,16 +1,18 @@
-{ lib, stdenv, fetchFromGitHub, autoreconfHook }:
+{ lib, stdenv, fetchurl, autoreconfHook }:
 
 stdenv.mkDerivation rec {
-  pname = "onig";
+  pname = "oniguruma";
   version = "6.9.8";
 
-  src = fetchFromGitHub {
-    owner = "kkos";
-    repo = "oniguruma";
-    rev = "v${version}";
-    sha256 = "sha256-8aFZdhh6ovLCR0A17rvWq/Oif66rSMnHcCYHjClNElw=";
+  # Note: do not use fetchpatch or fetchFromGitHub to keep this package available in __bootPackages
+  src = fetchurl {
+    url = "https://github.com/kkos/oniguruma/releases/download/v${version}/onig-${version}.tar.gz";
+    sha256 = "sha256-KM1iwUZGI8eRBWX7HMqqAQSy/osSvNZG6B9ztHU1IT4=";
   };
 
+  outputs = [ "dev" "lib" "out" ];
+  outputBin = "dev"; # onig-config
+
   nativeBuildInputs = [ autoreconfHook ];
   configureFlags = [ "--enable-posix-api=yes" ];
 
@@ -18,7 +20,7 @@ stdenv.mkDerivation rec {
     homepage = "https://github.com/kkos/oniguruma";
     description = "Regular expressions library";
     license = licenses.bsd2;
-    maintainers = with maintainers; [ ];
+    maintainers = with maintainers; [ artturin ];
     platforms = platforms.unix;
   };
 }

pkgs/development/tools/jq/default.nix

@@ -1,7 +1,6 @@
 { lib
 , stdenv
-, fetchpatch
-, fetchFromGitHub
+, fetchurl
 , autoreconfHook
 , onigurumaSupport ? true
 , oniguruma
@@ -11,19 +10,14 @@ stdenv.mkDerivation rec {
   pname = "jq";
   version = "1.6";
 
-  src = fetchFromGitHub {
-    owner = "stedolan";
-    repo = "jq";
-    rev = "${pname}-${version}";
-    hash = "sha256-CIE8vumQPGK+TFAncmpBijANpFALLTadOvkob0gVzro";
+  # Note: do not use fetchpatch or fetchFromGitHub to keep this package available in __bootPackages
+  src = fetchurl {
+    url = "https://github.com/stedolan/jq/releases/download/jq-${version}/jq-${version}.tar.gz";
+    sha256 = "sha256-XejI4pqqP7nMa0e7JymfJxNU67clFOOsytx9OLW7qnI=";
   };
 
   patches = [
-    (fetchpatch {
-      name = "fix-tests-when-building-without-regex-supports.patch";
-      url = "https://github.com/stedolan/jq/pull/2292/commits/f6a69a6e52b68a92b816a28eb20719a3d0cb51ae.patch";
-      sha256 = "pTM5FZ6hFs5Rdx+W2dICSS2lcoLY1Q//Lan3Hu8Gr58=";
-    })
+    ./fix-tests-when-building-without-regex-supports.patch
   ];
 
   outputs = [ "bin" "doc" "man" "dev" "lib" "out" ];
@@ -69,7 +63,7 @@ stdenv.mkDerivation rec {
     description = "A lightweight and flexible command-line JSON processor";
     homepage = "https://stedolan.github.io/jq/";
     license = licenses.mit;
-    maintainers = with maintainers; [ raskin globin ];
+    maintainers = with maintainers; [ raskin globin artturin ];
     platforms = platforms.unix;
     downloadPage = "https://stedolan.github.io/jq/download/";
   };

pkgs/development/tools/jq/fix-tests-when-building-without-regex-supports.patch

@@ -0,0 +1,38 @@
+From f6a69a6e52b68a92b816a28eb20719a3d0cb51ae Mon Sep 17 00:00:00 2001
+From: Dmitry Bogatov <git#v1@kaction.cc>
+Date: Sat, 27 Mar 2021 00:00:00 +0000
+Subject: [PATCH] Disable some tests when building without regex support
+
+---
+ Makefile.am  | 5 ++++-
+ configure.ac | 1 +
+ 2 files changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile.am b/Makefile.am
+index f5c1db594..f3f44bb9e 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -130,7 +130,10 @@ endif
+ 
+ ### Tests (make check)
+ 
+-TESTS = tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest tests/utf8test tests/base64test
++TESTS = tests/optionaltest tests/jqtest tests/shtest tests/utf8test tests/base64test
++if WITH_ONIGURUMA
++TESTS += tests/mantest tests/onigtest
++endif
+ TESTS_ENVIRONMENT = NO_VALGRIND=$(NO_VALGRIND)
+ 
+ # This is a magic make variable that causes it to treat tests/man.test as a
+diff --git a/configure.ac b/configure.ac
+index 0441d4a2c..987d94e0a 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -278,6 +278,7 @@ AC_SUBST(onig_CFLAGS)
+ AC_SUBST(onig_LDFLAGS)
+ 
+ AM_CONDITIONAL([BUILD_ONIGURUMA], [test "x$build_oniguruma" = xyes])
++AM_CONDITIONAL([WITH_ONIGURUMA], [test "x$with_oniguruma" = xyes])
+ AC_SUBST([BUNDLER], ["$bundle_cmd"])
+ 
+ AC_CONFIG_MACRO_DIR([config/m4])

pkgs/stdenv/generic/default.nix

@@ -57,28 +57,21 @@ argsStdenv@{ name ? "stdenv", preHook ? "", initialPath
 
 let
   defaultNativeBuildInputs = extraNativeBuildInputs ++
-    [ ../../build-support/setup-hooks/move-docs.sh
-      ../../build-support/setup-hooks/make-symlinks-relative.sh
+    [
+      ../../build-support/setup-hooks/audit-tmpdir.sh
       ../../build-support/setup-hooks/compress-man-pages.sh
-      ../../build-support/setup-hooks/strip.sh
+      ../../build-support/setup-hooks/make-symlinks-relative.sh
+      ../../build-support/setup-hooks/move-docs.sh
+      ../../build-support/setup-hooks/move-lib64.sh
+      ../../build-support/setup-hooks/move-sbin.sh
+      ../../build-support/setup-hooks/move-systemd-user-units.sh
+      ../../build-support/setup-hooks/multiple-outputs.sh
       ../../build-support/setup-hooks/patch-shebangs.sh
       ../../build-support/setup-hooks/prune-libtool-files.sh
-    ]
-      # FIXME this on Darwin; see
-      # https://github.com/NixOS/nixpkgs/commit/94d164dd7#commitcomment-22030369
-    ++ lib.optionals hostPlatform.isLinux [
-      ../../build-support/setup-hooks/audit-tmpdir.sh
-      ../../build-support/setup-hooks/move-systemd-user-units.sh
-    ]
-    ++ [
-      ../../build-support/setup-hooks/multiple-outputs.sh
-      ../../build-support/setup-hooks/move-sbin.sh
-      ../../build-support/setup-hooks/move-lib64.sh
-      ../../build-support/setup-hooks/set-source-date-epoch-to-latest.sh
       ../../build-support/setup-hooks/reproducible-builds.sh
-      # TODO use lib.optional instead
-      (if hasCC then cc else null)
-    ];
+      ../../build-support/setup-hooks/set-source-date-epoch-to-latest.sh
+      ../../build-support/setup-hooks/strip.sh
+    ] ++ lib.optionals hasCC [ cc ];
 
   defaultBuildInputs = extraBuildInputs;
 

pkgs/test/hooks/default.nix

@@ -1,28 +1,8 @@
 # To run these tests:
 # nix-build -A tests.hooks
 
-{ stdenv, pkgs, lib }:
+{ stdenv, tests, lib }:
 
 {
-  # this attrset is for hooks in `stdenv.defaultNativeBuildInputs`
-  default-stdenv-hooks = lib.recurseIntoAttrs {
-    make-symlinks-relative = stdenv.mkDerivation {
-      name = "test-make-symlinks-relative";
-      passAsFile = [ "buildCommand" ];
-      buildCommand = ''
-        mkdir -p $out/{bar,baz}
-        source1="$out/bar/foo"
-        destination1="$out/baz/foo"
-        echo foo > $source1
-        ln -s $source1 $destination1
-        echo "symlink before patching: $(readlink $destination1)"
-
-        _makeSymlinksRelative
-
-        echo "symlink after patching: $(readlink $destination1)"
-        ([[ -e $destination1 ]] && echo "symlink isn't broken") || (echo "symlink is broken" && exit 1)
-        ([[ $(readlink $destination1) == "../bar/foo" ]] && echo "absolute symlink was made relative") || (echo "symlink was not made relative" && exit 1)
-      '';
-    };
-  };
+  default-stdenv-hooks = lib.recurseIntoAttrs tests.stdenv.hooks;
 }

pkgs/test/stdenv/default.nix

@@ -8,6 +8,8 @@
 }:
 
 let
+  # early enough not to rebuild gcc but late enough to have patchelf
+  earlyPkgs = stdenv.__bootPackages.stdenv.__bootPackages;
   # use a early stdenv so when hacking on stdenv this test can be run quickly
   bootStdenv = stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv;
   pkgsStructured = import pkgs.path { config = { structuredAttrsByDefault = true; }; inherit (stdenv.hostPlatform) system; };
@@ -93,6 +95,9 @@ let
 in
 
 {
+  # tests for hooks in `stdenv.defaultNativeBuildInputs`
+  hooks = lib.recurseIntoAttrs (import ./hooks.nix { stdenv = bootStdenv; pkgs = earlyPkgs; });
+
   test-env-attrset = testEnvAttrset { name = "test-env-attrset"; stdenv' = bootStdenv; };
 
   test-prepend-append-to-var = testPrependAndAppendToVar {
@@ -112,6 +117,9 @@ in
   };
 
   structuredAttrsByDefault = lib.recurseIntoAttrs {
+
+    hooks = lib.recurseIntoAttrs (import ./hooks.nix { stdenv = bootStdenvStructuredAttrsByDefault; pkgs = earlyPkgs; });
+
     test-cc-wrapper-substitutions = ccWrapperSubstitutionsTest {
       name = "test-cc-wrapper-substitutions-structuredAttrsByDefault";
       stdenv' = bootStdenvStructuredAttrsByDefault;
@@ -155,5 +163,74 @@ in
         '';
       };
     };
+
+    test-golden-example-structuredAttrs =
+      let
+        goldenSh = earlyPkgs.writeText "goldenSh" ''
+          declare -A EXAMPLE_ATTRS=(['foo']='bar' )
+          declare EXAMPLE_BOOL_FALSE=
+          declare EXAMPLE_BOOL_TRUE=1
+          declare EXAMPLE_INT=123
+          declare EXAMPLE_INT_NEG=-123
+          declare -a EXAMPLE_LIST=('foo' 'bar' )
+          declare EXAMPLE_STR='foo bar'
+        '';
+        goldenJson = earlyPkgs.writeText "goldenSh" ''
+          {
+            "EXAMPLE_ATTRS": {
+              "foo": "bar"
+            },
+            "EXAMPLE_BOOL_FALSE": false,
+            "EXAMPLE_BOOL_TRUE": true,
+            "EXAMPLE_INT": 123,
+            "EXAMPLE_INT_NEG": -123,
+            "EXAMPLE_LIST": [
+              "foo",
+              "bar"
+            ],
+            "EXAMPLE_NESTED_ATTRS": {
+              "foo": {
+                "bar": "baz"
+              }
+            },
+            "EXAMPLE_NESTED_LIST": [
+              [
+                "foo",
+                "bar"
+              ],
+              [
+                "baz"
+              ]
+            ],
+            "EXAMPLE_STR": "foo bar"
+          }
+        '';
+      in
+      bootStdenvStructuredAttrsByDefault.mkDerivation {
+        name = "test-golden-example-structuredAttrsByDefault";
+        nativeBuildInputs = [ earlyPkgs.jq ];
+
+        EXAMPLE_BOOL_TRUE = true;
+        EXAMPLE_BOOL_FALSE = false;
+        EXAMPLE_INT = 123;
+        EXAMPLE_INT_NEG = -123;
+        EXAMPLE_STR = "foo bar";
+        EXAMPLE_LIST = [ "foo" "bar" ];
+        EXAMPLE_NESTED_LIST = [ [ "foo" "bar" ] [ "baz" ] ];
+        EXAMPLE_ATTRS = { foo = "bar"; };
+        EXAMPLE_NESTED_ATTRS = { foo.bar = "baz"; };
+
+        inherit goldenSh;
+        inherit goldenJson;
+
+        buildCommand = ''
+          mkdir -p $out
+          cat $NIX_ATTRS_SH_FILE | grep "EXAMPLE" | grep -v -E 'installPhase|jq' > $out/sh
+          jq 'with_entries(select(.key|match("EXAMPLE")))' $NIX_ATTRS_JSON_FILE > $out/json
+          diff $out/sh $goldenSh
+          diff $out/json $goldenJson
+        '';
+      };
+
   };
 }

pkgs/test/stdenv/hooks.nix

@@ -0,0 +1,126 @@
+{ stdenv, pkgs }:
+
+# ordering should match defaultNativeBuildInputs
+
+{
+  # TODO: add audit-tmpdir
+  compress-man-pages =
+    let
+      manFile = pkgs.writeText "small-man" ''
+        .TH HELLO "1" "May 2022" "hello 2.12.1" "User Commands"
+        .SH NAME
+        hello - friendly greeting program
+      '';
+    in
+    stdenv.mkDerivation {
+      name = "test-compress-man-pages";
+      buildCommand = ''
+        mkdir -p $out/share/man
+        cp ${manFile} $out/share/man/small-man.1
+        compressManPages $out
+        [[ -e $out/share/man/small-man.1.gz ]]
+      '';
+    };
+  make-symlinks-relative = stdenv.mkDerivation {
+    name = "test-make-symlinks-relative";
+    buildCommand = ''
+      mkdir -p $out/{bar,baz}
+      source1="$out/bar/foo"
+      destination1="$out/baz/foo"
+      echo foo > $source1
+      ln -s $source1 $destination1
+      echo "symlink before patching: $(readlink $destination1)"
+
+      _makeSymlinksRelative
+
+      echo "symlink after patching: $(readlink $destination1)"
+      ([[ -e $destination1 ]] && echo "symlink isn't broken") || (echo "symlink is broken" && exit 1)
+      ([[ $(readlink $destination1) == "../bar/foo" ]] && echo "absolute symlink was made relative") || (echo "symlink was not made relative" && exit 1)
+    '';
+  };
+  move-docs = stdenv.mkDerivation {
+    name = "test-move-docs";
+    buildCommand = ''
+      mkdir -p $out/{man,doc,info}
+      touch $out/{man,doc,info}/foo
+      cat $out/{man,doc,info}/foo
+
+      _moveToShare
+
+      (cat $out/share/{man,doc,info}/foo 2>/dev/null && echo "man,doc,info were moved") || (echo "man,doc,info were not moved" && exit 1)
+    '';
+  };
+  move-lib64 = stdenv.mkDerivation {
+    name = "test-move-lib64";
+    buildCommand = ''
+      mkdir -p $out/lib64
+      touch $out/lib64/foo
+      cat $out/lib64/foo
+
+      _moveLib64
+
+      # check symlink
+      [[ -h $out/lib64 ]]
+      ([[ -e $out/lib64 ]] && echo "symlink isn't broken") || (echo "symlink is broken" && exit 1)
+      [[ -e $out/lib/foo ]]
+    '';
+  };
+  move-sbin = stdenv.mkDerivation {
+    name = "test-move-sbin";
+    buildCommand = ''
+      mkdir -p $out/sbin
+      touch $out/sbin/foo
+      cat $out/sbin/foo
+
+      _moveSbin
+
+      # check symlink
+      [[ -h $out/sbin ]]
+      ([[ -e $out/sbin ]] && echo "symlink isn't broken") || (echo "symlink is broken" && exit 1)
+      [[ -e $out/bin/foo ]]
+    '';
+  };
+  # TODO: add multiple-outputs
+  # TODO: move patch-shebangs test from pkgs/test/patch-shebangs/default.nix to here
+  prune-libtool-files =
+    let
+      libFoo = pkgs.writeText "libFoo" ''
+        # Generated by libtool (GNU libtool) 2.4.6
+        old_library='''
+        dependency_libs=' -Lbar.la -Lbaz.la'
+      '';
+    in
+    stdenv.mkDerivation {
+      name = "test-prune-libtool-files";
+      buildCommand = ''
+        mkdir -p $out/lib
+        cp ${libFoo} $out/lib/libFoo.la
+        _pruneLibtoolFiles
+        grep "^dependency_libs=''' #pruned" $out/lib/libFoo.la
+        # confirm file doesn't only contain the above
+        grep "^old_library='''" $out/lib/libFoo.la
+      '';
+    };
+  reproducible-builds = stdenv.mkDerivation {
+    name = "test-reproducible-builds";
+    buildCommand = ''
+      # can't be tested more precisely because the value of random-seed changes depending on the output
+      [[ $NIX_CFLAGS_COMPILE =~ "-frandom-seed=" ]]
+      touch $out
+    '';
+  };
+  set-source-date-epoch-to-latest = stdenv.mkDerivation {
+    name = "test-set-source-date-epoch-to-latest";
+    buildCommand = ''
+      sourceRoot=$NIX_BUILD_TOP/source
+      mkdir -p $sourceRoot
+      touch --date=1/1/2015 $sourceRoot/foo
+
+      _updateSourceDateEpochFromSourceRoot
+
+      [[ $SOURCE_DATE_EPOCH == "1420070400" ]]
+      touch $out
+    '';
+  };
+  # TODO: add strip
+}