stdenv.mkDerivation: Allow overriding of recursive definitions

See updated manual for further explanation.

doc/stdenv/meta.chapter.md

@@ -175,6 +175,36 @@ The NixOS tests are available as `nixosTests` in parameters of derivations. For
 
 NixOS tests run in a VM, so they are slower than regular package tests. For more information see [NixOS module tests](https://nixos.org/manual/nixos/stable/#sec-nixos-tests).
 
+Alternatively, you can specify other derivations as tests. You can make use of
+the optional parameter (here: `self`) to inject the correct package without
+relying on non-local definitions, even in the presence of `overrideAttrs`. This
+definition of `tests` does not rely on the original `mypkg` or overrides it in
+all places.
+
+```nix
+# my-package/default.nix
+{ stdenv, callPackage }:
+stdenv.mkDerivation (self: {
+  # ...
+  passthru.tests.example = callPackage ./example.nix { my-package = self; };
+})
+```
+
+```nix
+# my-package/example.nix
+{ runCommand, lib, my-package, ... }:
+runCommand "my-package-test" {
+  nativeBuildInputs = [ my-package ];
+  src = lib.sources.sourcesByRegex ./. [ ".*.in" ".*.expected" ];
+} ''
+  my-package --help
+  my-package <example.in >example.actual
+  diff -U3 --color=auto example.expected example.actual
+  mkdir $out
+''
+```
+
+
 ### `timeout` {#var-meta-timeout}
 
 A timeout (in seconds) for building the derivation. If the derivation takes longer than this time to build, it can fail due to breaking the timeout. However, all computers do not have the same computing power, hence some builders may decide to apply a multiplicative factor to this value. When filling this value in, try to keep it approximately consistent with other values already present in `nixpkgs`.

doc/stdenv/stdenv.chapter.md

@@ -317,6 +317,58 @@ The script will be usually run from the root of the Nixpkgs repository but you s
 
 For information about how to run the updates, execute `nix-shell maintainers/scripts/update.nix`.
 
+### Recursive attributes in `mkDerivation`
+
+If you pass a function to `mkDerivation`, it will receive as its argument the final output of the same `mkDerivation` call. For example:
+
+```nix
+mkDerivation (self: {
+  pname = "hello";
+  withFeature = true;
+  configureFlags =
+    lib.optionals self.withFeature ["--with-feature"];
+})
+```
+
+Note that this does not use the `rec` keyword to reuse `withFeature` in `configureFlags`.
+Instead, the definition references `self`, allowing users to change `withFeature`
+consistently with `overrideAttrs`.
+
+Let's look at a more elaborate example to understand the differences between
+various bindings:
+
+```nix
+# `pkg` is the _original_ definition (for illustration purposes)
+let pkg =
+  mkDerivation (self: {   # self is the final package
+    # ...
+
+    # An example attribute
+    packages = [];
+
+    # `passthru.tests` is a commonly defined attribute.
+    passthru.tests.simple = f self;
+
+    # An example of an attribute containing a function
+    passthru.appendPackages = packages':
+      self.overrideAttrs (newSelf: super: {
+        packages = super.packages ++ packages';
+      });
+
+    # For illustration purposes; referenced as
+    # `(pkg.overrideAttrs(x)).self` etc in the text below.
+    passthru.self = self;
+    passthru.original = pkg;
+  });
+in pkg
+```
+
+Unlike the `pkg` binding in the above example, the `self` parameter always references the final package. For instance `(pkg.overrideAttrs(x)).self` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as `pkg`.
+
+This is also different from `mkDerivation rec { ..... }`, which binds the recursive references immediately, so it allows you to reference original _inputs_ only.
+
+See also the section about [`passthru.tests`](#var-meta-tests).
+
 ## Phases {#sec-stdenv-phases}
 
 `stdenv.mkDerivation` sets the Nix [derivation](https://nixos.org/manual/nix/stable/expressions/derivations.html#derivations)'s builder to a script that loads the stdenv `setup.sh` bash library and calls `genericBuild`. Most packaging functions rely on this default builder.

pkgs/stdenv/generic/make-derivation.nix

@@ -9,8 +9,68 @@ let
     # to build it. This is a bit confusing for cross compilation.
     inherit (stdenv) hostPlatform;
   };
+
+  makeOverlayable = mkDerivationSimple: # TODO(@robert): turn mkDerivationSimple into let binding.
+    fnOrAttrs:
+      if builtins.isFunction fnOrAttrs
+      then makeDerivationExtensible mkDerivationSimple fnOrAttrs
+      else makeDerivationExtensibleConst mkDerivationSimple fnOrAttrs;
+
+  # Based off lib.makeExtensible, with modifications:
+  #  - lib.fix' -> lib.fix ∘ mkDerivationSimple; then inline fix
+  #  - convert `f` to an overlay
+  #  - inline overrideAttrs and make it positional instead of // to reduce allocs
+  makeDerivationExtensible = mkDerivationSimple: rattrs:
+    let
+      r = mkDerivationSimple
+        (f0:
+          let
+            f = self: super:
+              # Convert f0 to an overlay. Legacy is:
+              #   overrideAttrs (super: {})
+              # We want to introduce self. We follow the convention of overlays:
+              #   overrideAttrs (self: super: {})
+              # Which means the first parameter can be either self or super.
+              # This is surprising, but far better than the confusion that would
+              # arise from flipping an overlay's parameters in some cases.
+              let x = f0 super;
+              in
+                if builtins.isFunction x
+                then
+                  # Can't reuse `x`, because `self` comes first.
+                  # Looks inefficient, but `f0 super` was a cheap thunk.
+                  f0 self super
+                else x;
+          in
+            makeDerivationExtensible mkDerivationSimple
+              (self: let super = rattrs self; in super // f self super))
+        (rattrs r);
+    in r;
+
+  # makeDerivationExtensibleConst == makeDerivationExtensible (_: attrs),
+  # but pre-evaluated for a slight improvement in performance.
+  makeDerivationExtensibleConst = mkDerivationSimple: attrs:
+    mkDerivationSimple (f0:
+      let
+        f = self: super:
+          let x = f0 super;
+          in
+            if builtins.isFunction x
+            then
+              # Can't reuse `x`, because `self` comes first.
+              # Looks inefficient, but `f0 super` was a cheap thunk.
+              f0 self super
+            else x;
+      in
+        makeDerivationExtensible mkDerivationSimple (self: attrs // f self attrs))
+      attrs;
+
 in
 
+# TODO(@roberth): inline makeOverlayable; reindenting whole rest of this file.
+makeOverlayable (overrideAttrs:
+
+
 # `mkDerivation` wraps the builtin `derivation` function to
 # produce derivations that use this stdenv and its shell.
 #
@@ -70,6 +130,7 @@ in
 
 , # TODO(@Ericson2314): Make always true and remove
   strictDeps ? if config.strictDepsByDefault then true else stdenv.hostPlatform != stdenv.buildPlatform
+
 , meta ? {}
 , passthru ? {}
 , pos ? # position used in error messages and for meta.position
@@ -381,8 +442,6 @@ in
 lib.extendDerivation
   validity.handled
   ({
-     overrideAttrs = f: stdenv.mkDerivation (attrs // (f attrs));
-
      # A derivation that always builds successfully and whose runtime
      # dependencies are the original derivations build time dependencies
      # This allows easy building and distributing of all derivations
@@ -408,10 +467,12 @@ lib.extendDerivation
        args = [ "-c" "export > $out" ];
      });
 
-     inherit meta passthru;
+     inherit meta passthru overrideAttrs;
    } //
    # Pass through extra attributes that are not inputs, but
    # should be made available to Nix expressions using the
    # derivation (e.g., in assertions).
    passthru)
   (derivation derivationArg)
+
+)