neovimUtils: neovim utilities to handle more usecases

Current nixpkgs always wraps neovim with the "-u" which has sideeffects as explained in https://github.com/NixOS/nixpkgs/issues/55376 : 
1.  vim won't set the variable $MYVIMRC as explained #34215
 2. vim skips loading folder-specific .vimrc / .nvimrc

I wanted to provide a way for users to better control what flags are used to wrap neovim. This is achived by introducing wrapNeovimUnstable et neovimUtils, utilities to help with that. We provide a compatibility layer so that wrapNeovim still works and to let us experiment with wrapNeovimUnstable to better control neovim configuration, plugin dependencies, haskell environment etc so that it becomes easier to generate per-project neovim config.

With this commit, it's possible for instance for home-manager to wrap neovim without the `-u` and just write the config in the
expected $XDG_CONFIG_HOME/nvim/init.vim .
Expect wrapNeovimUnstable interface to evolve in the upcoming months.

pkgs/applications/editors/neovim/utils.nix

@@ -0,0 +1,163 @@
+{ lib
+, vimUtils
+, nodejs
+, neovim-unwrapped
+, bundlerEnv
+, ruby
+, pythonPackages
+, python3Packages
+, writeText
+, wrapNeovimUnstable
+}:
+let
+  # returns everything needed for the caller to wrap its own neovim:
+  # - the generated content of the future init.vim
+  # - the arguments to wrap neovim with
+  # The caller is responsible for writing the init.vim and adding it to the wrapped
+  # arguments (["-u" writeText "init.vim" GENERATEDRC)]).
+  # This makes it possible to write the config anywhere: on a per-project basis
+  # .nvimrc or in $XDG_CONFIG_HOME/nvim/init.vim to avoid sideeffects.
+  # Indeed, note that wrapping with `-u init.vim` has sideeffects like .nvimrc wont be loaded
+  # anymore, $MYVIMRC wont be set etc
+  makeNeovimConfig =
+    {
+    withPython2 ? false
+    /* the function you would have passed to python.withPackages */
+    , extraPython2Packages ? (_: [ ])
+    ,  withPython3 ? true
+    /* the function you would have passed to python3.withPackages */
+    , extraPython3Packages ? (_: [ ])
+    , withNodeJs ? false
+    , withRuby ? true
+
+    # same values as in vimUtils.vimrcContent
+    , configure ? { }
+
+    # for forward compability, when adding new environments, haskell etc.
+    , ...
+    }:
+    let
+      rubyEnv = bundlerEnv {
+        name = "neovim-ruby-env";
+        gemdir = ./ruby_provider;
+        postBuild = ''
+          ln -sf ${ruby}/bin/* $out/bin
+        '';
+      };
+
+
+      requiredPlugins = vimUtils.requiredPlugins configure;
+      getDeps = attrname: map (plugin: plugin.${attrname} or (_: [ ]));
+
+      pluginPython2Packages = getDeps "pythonDependencies" requiredPlugins;
+      python2Env = pythonPackages.python.withPackages (ps:
+        [ ps.pynvim ]
+        ++ (extraPython2Packages ps)
+        ++ (lib.concatMap (f: f ps) pluginPython2Packages));
+
+      pluginPython3Packages = getDeps "python3Dependencies" requiredPlugins;
+      python3Env = python3Packages.python.withPackages (ps:
+        [ ps.pynvim ]
+        ++ (extraPython3Packages ps)
+        ++ (lib.concatMap (f: f ps) pluginPython3Packages));
+
+
+      # Mapping a boolean argument to a key that tells us whether to add or not to
+      # add to nvim's 'embedded rc' this:
+      #    let g:<key>_host_prog=$out/bin/nvim-<key>
+      # Or this:
+      #    let g:loaded_${prog}_provider=1
+      # While the latter tells nvim that this provider is not available
+      hostprog_check_table = {
+        node = withNodeJs;
+        python = withPython2;
+        python3 = withPython3;
+        ruby = withRuby;
+      };
+      ## Here we calculate all of the arguments to the 1st call of `makeWrapper`
+      # We start with the executable itself NOTE we call this variable "initial"
+      # because if configure != {} we need to call makeWrapper twice, in order to
+      # avoid double wrapping, see comment near finalMakeWrapperArgs
+      makeWrapperArgs =
+        let
+          binPath = lib.makeBinPath (lib.optionals withRuby [ rubyEnv ] ++ lib.optionals withNodeJs [ nodejs ]);
+
+          flags = lib.concatLists (lib.mapAttrsToList (
+              prog: withProg: [
+                "--cmd" (genProviderSettings prog withProg)
+              ]
+            )
+            hostprog_check_table);
+        in
+        [
+          "--argv0" "$0" "--add-flags" (lib.escapeShellArgs flags)
+        ] ++ lib.optionals withRuby [
+          "--set" "GEM_HOME" "${rubyEnv}/${rubyEnv.ruby.gemPath}"
+        ] ++ lib.optionals (binPath != "") [
+          "--suffix" "PATH" ":" binPath
+        ];
+
+      manifestRc = vimUtils.vimrcContent (configure // { customRC = ""; });
+      neovimRcContent = vimUtils.vimrcContent configure;
+    in
+    {
+      wrapperArgs = makeWrapperArgs;
+      inherit neovimRcContent;
+      inherit manifestRc;
+      inherit rubyEnv;
+      inherit python2Env;
+      inherit python3Env;
+    };
+
+    genProviderSettings = prog: withProg:
+      if withProg then
+        "let g:${prog}_host_prog='${placeholder "out"}/bin/nvim-${prog}'"
+      else
+        "let g:loaded_${prog}_provider=1"
+    ;
+
+  # to keep backwards compatibility
+  legacyWrapper = neovim: {
+    extraMakeWrapperArgs ? []
+    , withPython ? true
+    /* the function you would have passed to python.withPackages */
+    , extraPythonPackages ? (_: [])
+    /* the function you would have passed to python.withPackages */
+    , withPython3 ? true,  extraPython3Packages ? (_: [])
+    , withNodeJs ? false
+    , withRuby ? true
+    , vimAlias ? false
+    , viAlias ? false
+    , configure ? {}
+  }:
+    let
+      /* for compatibility with passing extraPythonPackages as a list; added 2018-07-11 */
+      compatFun = funOrList: (if builtins.isList funOrList then
+        (_: lib.warn "passing a list as extraPythonPackages to the neovim wrapper is deprecated, pass a function as to python.withPackages instead" funOrList)
+      else funOrList);
+
+      res = makeNeovimConfig {
+        withPython2 = withPython;
+        extraPythonPackages = compatFun extraPythonPackages;
+        inherit withPython3;
+        extraPython3Packages = compatFun extraPython3Packages;
+        inherit withNodeJs withRuby;
+
+        inherit configure;
+      };
+    in
+    wrapNeovimUnstable neovim (res // {
+      wrapperArgs = res.wrapperArgs
+      ++ [
+        "--add-flags" "-u ${writeText "init.vim" res.neovimRcContent}"
+      ]
+      ++ (if builtins.isList extraMakeWrapperArgs then extraMakeWrapperArgs
+      else lib.warn "Passing a string as extraMakeWrapperArgs to the neovim wrapper is
+        deprecated, please use a list instead")
+      ;
+  });
+in
+{
+  inherit makeNeovimConfig;
+  inherit legacyWrapper;
+}

pkgs/applications/editors/neovim/wrapper.nix

@@ -13,124 +13,41 @@ neovim:
 
 let
   wrapper = {
-      extraMakeWrapperArgs ? ""
+      # should contain all args but the binary
-    , withPython ? true,  extraPythonPackages ? (_: []) /* the function you would have passed to python.withPackages */
+      wrapperArgs ? []
-    , withPython3 ? true,  extraPython3Packages ? (_: []) /* the function you would have passed to python.withPackages */
+    , manifestRc ? null
+    , withPython2 ? true, python2Env ? null
+    , withPython3 ? true,  python3Env ? null
     , withNodeJs? false
-    , withRuby ? true
+    , withRuby ? true, rubyEnv ? null
     , vimAlias ? false
     , viAlias ? false
-    , configure ? {}
+    , ...
   }:
   let
 
-  rubyEnv = bundlerEnv {
-    name = "neovim-ruby-env";
-    gemdir = ./ruby_provider;
-    postBuild = ''
-      ln -sf ${ruby}/bin/* $out/bin
-    '';
-  };
-
-  /* for compatibility with passing extraPythonPackages as a list; added 2018-07-11 */
-  compatFun = funOrList: (if builtins.isList funOrList then
-    (_: lib.warn "passing a list as extraPythonPackages to the neovim wrapper is deprecated, pass a function as to python.withPackages instead" funOrList)
-    else funOrList);
-  extraPythonPackagesFun = compatFun extraPythonPackages;
-  extraPython3PackagesFun = compatFun extraPython3Packages;
-
-  requiredPlugins = vimUtils.requiredPlugins configure;
-  getDeps = attrname: map (plugin: plugin.${attrname} or (_:[]));
-
-  pluginPythonPackages = getDeps "pythonDependencies" requiredPlugins;
-  pythonEnv = pythonPackages.python.withPackages(ps:
-        [ ps.pynvim ]
-        ++ (extraPythonPackagesFun ps)
-        ++ (concatMap (f: f ps) pluginPythonPackages));
-
-  pluginPython3Packages = getDeps "python3Dependencies" requiredPlugins;
-  python3Env = python3Packages.python.withPackages (ps:
-        [ ps.pynvim ]
-        ++ (extraPython3PackagesFun ps)
-        ++ (concatMap (f: f ps) pluginPython3Packages));
-
-  binPath = makeBinPath (optionals withRuby [rubyEnv] ++ optionals withNodeJs [nodejs]);
-
-  # Mapping a boolean argument to a key that tells us whether to add or not to
-  # add to nvim's 'embedded rc' this:
-  #
-  #    let g:<key>_host_prog=$out/bin/nvim-<key>
-  #
-  # Or this:
-  #
-  #    let g:loaded_${prog}_provider=1
-  #
-  # While the later tells nvim that this provider is not available
-  #
-  hostprog_check_table = {
-    node = withNodeJs;
-    python = withPython;
-    python3 = withPython3;
-    ruby = withRuby;
-  };
-  ## Here we calculate all of the arguments to the 1st call of `makeWrapper`
-  # We start with the executable itself NOTE we call this variable "initial"
-  # because if configure != {} we need to call makeWrapper twice, in order to
-  # avoid double wrapping, see comment near finalMakeWrapperArgs
-  initialMakeWrapperArgs =
-    let
-      flags = lib.concatLists (lib.mapAttrsToList (
-        prog:
-        withProg:
-        [
-          "--cmd"
-          (if withProg then
-            "let g:${prog}_host_prog='${placeholder "out"}/bin/nvim-${prog}'"
-          else
-            "let g:loaded_${prog}_provider=1"
-            )
-        ]
-      ) hostprog_check_table);
-    in [
-      "${neovim}/bin/nvim" "${placeholder "out"}/bin/nvim"
-      "--argv0" "$0"
-      "--add-flags" (lib.escapeShellArgs flags)
-    ] ++ lib.optionals withRuby [
-      "--set" "GEM_HOME" "${rubyEnv}/${rubyEnv.ruby.gemPath}"
-    ] ++ lib.optionals (binPath != "") [
-      "--suffix" "PATH" ":" binPath
-    ];
   # If configure != {}, we can't generate the rplugin.vim file with e.g
   # NVIM_SYSTEM_RPLUGIN_MANIFEST *and* NVIM_RPLUGIN_MANIFEST env vars set in
   # the wrapper. That's why only when configure != {} (tested both here and
   # when postBuild is evaluated), we call makeWrapper once to generate a
   # wrapper with most arguments we need, excluding those that cause problems to
   # generate rplugin.vim, but still required for the final wrapper.
-  finalMakeWrapperArgs = initialMakeWrapperArgs
+  finalMakeWrapperArgs =
-    # this relies on a patched neovim, see
+    [ "${neovim}/bin/nvim" "${placeholder "out"}/bin/nvim" ] ++ wrapperArgs ++
-    # https://github.com/neovim/neovim/issues/9413
+      [ "--set" "NVIM_SYSTEM_RPLUGIN_MANIFEST" "${placeholder "out"}/rplugin.vim" ];
-    ++ lib.optionals (configure != {}) [
-      "--set" "NVIM_SYSTEM_RPLUGIN_MANIFEST" "${placeholder "out"}/rplugin.vim"
-      "--add-flags" "-u ${configFile}"
-    ];
-   configFile = writeText "init.vim" "${vimUtils.vimrcContent configure}";
   in
   symlinkJoin {
       name = "neovim-${stdenv.lib.getVersion neovim}";
       # Remove the symlinks created by symlinkJoin which we need to perform
       # extra actions upon
-      postBuild = ''
+      postBuild = lib.optionalString stdenv.isLinux ''
-        rm $out/bin/nvim
-        makeWrapper ${lib.escapeShellArgs initialMakeWrapperArgs} ${extraMakeWrapperArgs}
-      ''
-      + lib.optionalString stdenv.isLinux ''
         rm $out/share/applications/nvim.desktop
         substitute ${neovim}/share/applications/nvim.desktop $out/share/applications/nvim.desktop \
           --replace 'TryExec=nvim' "TryExec=$out/bin/nvim" \
           --replace 'Name=Neovim' 'Name=WrappedNeovim'
       ''
-      + optionalString withPython ''
+      + optionalString withPython2 ''
-        makeWrapper ${pythonEnv}/bin/python $out/bin/nvim-python --unset PYTHONPATH
+        makeWrapper ${python2Env}/bin/python $out/bin/nvim-python --unset PYTHONPATH
       ''
       + optionalString withPython3 ''
         makeWrapper ${python3Env}/bin/python3 $out/bin/nvim-python3 --unset PYTHONPATH
@@ -147,9 +64,14 @@ let
       + optionalString viAlias ''
         ln -s $out/bin/nvim $out/bin/vi
       ''
-      + optionalString (configure != {}) ''
+      + optionalString (manifestRc != null) (let
+        manifestWrapperArgs =
+          [ "${neovim}/bin/nvim" "${placeholder "out"}/bin/nvim-wrapper" ] ++ wrapperArgs;
+      in ''
         echo "Generating remote plugin manifest"
         export NVIM_RPLUGIN_MANIFEST=$out/rplugin.vim
+        makeWrapper ${lib.escapeShellArgs manifestWrapperArgs}
+
         # Some plugins assume that the home directory is accessible for
         # initializing caches, temporary files, etc. Even if the plugin isn't
         # actively used, it may throw an error as soon as Neovim is launched
@@ -165,8 +87,8 @@ let
         # (swap/viminfo) and redirect errors to stderr.
         # Only display the log on error since it will contain a few normally
         # irrelevant messages.
-        if ! $out/bin/nvim \
+        if ! $out/bin/nvim-wrapper \
-          -u ${vimUtils.vimrcFile (configure // { customRC = ""; })} \
+          -u ${writeText "manifest.vim" manifestRc} \
           -i NONE -n \
           -E -V1rplugins.log -s \
           +UpdateRemotePlugins +quit! > outfile 2>&1; then
@@ -174,7 +96,11 @@ let
           echo -e "\nGenerating rplugin.vim failed!"
           exit 1
         fi
-        makeWrapper ${lib.escapeShellArgs finalMakeWrapperArgs} ${extraMakeWrapperArgs}
+        rm "${placeholder "out"}/bin/nvim-wrapper"
+      '')
+      + ''
+        rm $out/bin/nvim
+        makeWrapper ${lib.escapeShellArgs finalMakeWrapperArgs}
       '';
 
     paths = [ neovim ];
@@ -182,7 +108,7 @@ let
     preferLocalBuild = true;
 
     buildInputs = [makeWrapper];
-    passthru = { unwrapped = neovim; inherit configFile; };
+    passthru = { unwrapped = neovim; };
 
     meta = neovim.meta // {
       # To prevent builds on hydra

pkgs/top-level/all-packages.nix

@@ -23972,8 +23972,11 @@ in
 
   vimpc = callPackage ../applications/audio/vimpc { };
 
-  wrapNeovim = callPackage ../applications/editors/neovim/wrapper.nix { };
+  # this is a lower-level alternative to wrapNeovim conceived to handle
-
+  # more usecases when wrapping neovim. The interface is being actively worked on
+  # so expect breakage. use wrapNeovim instead if you want a stable alternative
+  wrapNeovimUnstable = callPackage ../applications/editors/neovim/wrapper.nix { };
+  wrapNeovim = neovimUtils.legacyWrapper;
   neovim-unwrapped = callPackage ../applications/editors/neovim {
     lua =
       # neovim doesn't work with luajit on aarch64: https://github.com/neovim/neovim/issues/7879
@@ -23981,6 +23984,7 @@ in
       luajit;
   };
 
+  neovimUtils = callPackage ../applications/editors/neovim/utils.nix { };
   neovim = wrapNeovim neovim-unwrapped { };
 
   neovim-qt = libsForQt5.callPackage ../applications/editors/neovim/qt.nix { };