nix-gitignore: init at v3.0.0 (#46112)

closes siers/nix-gitignore#6

doc/functions.xml

@@ -17,4 +17,5 @@
  <xi:include href="functions/shell.xml" />
  <xi:include href="functions/dockertools.xml" />
  <xi:include href="functions/prefer-remote-fetch.xml" />
+ <xi:include href="functions/nix-gitignore.xml" />
 </chapter>

doc/functions/nix-gitignore.xml

@@ -0,0 +1,78 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xml:id="sec-pkgs-nix-gitignore">
+ <title>pkgs.nix-gitignore</title>
+
+ <para>
+  <function>pkgs.nix-gitignore</function> is a function that acts similarly to
+  <literal>builtins.filterSource</literal> but also allows filtering with the
+  help of the gitignore format.
+ </para>
+
+ <section xml:id="sec-pkgs-nix-gitignore-usage">
+  <title>Usage</title>
+
+  <para>
+  <literal>pkgs.nix-gitignore</literal> exports a number of functions, but
+  you'll most likely need either <literal>gitignoreSource</literal> or
+  <literal>gitignoreSourcePure</literal>. As their first argument, they both
+  accept either 1. a file with gitignore lines or 2. a string
+  with gitignore lines, or 3. a list of either of the two. They will be
+  concatenated into a single big string.
+  </para>
+
+  <programlisting><![CDATA[
+{ pkgs ? import <nixpkgs> {} }:
+
+ nix-gitignore.gitignoreSource [] ./source
+     # Simplest version
+
+ nix-gitignore.gitignoreSource "supplemental-ignores\n" ./source
+     # This one reads the ./source/.gitignore and concats the auxiliary ignores
+
+ nix-gitignore.gitignoreSourcePure "ignore-this\nignore-that\n" ./source
+     # Use this string as gitignore, don't read ./source/.gitignore.
+
+ nix-gitignore.gitignoreSourcePure ["ignore-this\nignore-that\n", ~/.gitignore] ./source
+     # It also accepts a list (of strings and paths) that will be concatenated
+     # once the paths are turned to strings via readFile.
+  ]]></programlisting>
+
+  <para>
+  These functions are derived from the <literal>Filter</literal> functions
+  by setting the first filter argument to <literal>(_: _: true)</literal>:
+  </para>
+
+  <programlisting><![CDATA[
+gitignoreSourcePure = gitignoreFilterSourcePure (_: _: true);
+gitignoreSource = gitignoreFilterSource (_: _: true);
+  ]]></programlisting>
+
+  <para>
+  Those filter functions accept the same arguments the <literal>builtins.filterSource</literal> function would pass to its filters, thus <literal>fn: gitignoreFilterSourcePure fn ""</literal> should be extensionally equivalent to <literal>filterSource</literal>. The file is blacklisted iff it's blacklisted by either your filter or the gitignoreFilter.
+  </para>
+
+  <para>
+  If you want to make your own filter from scratch, you may use
+ </para>
+
+  <programlisting><![CDATA[
+gitignoreFilter = ign: root: filterPattern (gitignoreToPatterns ign) root;
+  ]]></programlisting>
+ </section>
+
+ <section xml:id="sec-pkgs-nix-gitignore-usage-recursive">
+  <title>gitignore files in subdirectories</title>
+
+  <para>
+  If you wish to use a filter that would search for .gitignore files in subdirectories, just like git does by default, use this function:
+ </para>
+
+  <programlisting><![CDATA[
+gitignoreFilterRecursiveSource = filter: patterns: root:
+# OR
+gitignoreRecursiveSource = gitignoreFilterSourcePure (_: _: true);
+  ]]></programlisting>
+ </section>
+</section>

pkgs/build-support/nix-gitignore/default.nix

@@ -0,0 +1,178 @@
+# https://github.com/siers/nix-gitignore/
+
+{ lib, runCommand }:
+
+# An interesting bit from the gitignore(5):
+# - A slash followed by two consecutive asterisks then a slash matches
+# - zero or more directories. For example, "a/**/b" matches "a/b",
+# - "a/x/b", "a/x/y/b" and so on.
+
+with builtins;
+
+let
+  debug = a: trace a a;
+  last = l: elemAt l ((length l) - 1);
+
+  throwIfOldNix = let required = "2.0"; in
+    if compareVersions nixVersion required == -1
+    then throw "nix (v${nixVersion} =< v${required}) is too old for nix-gitignore"
+    else true;
+in rec {
+  # [["good/relative/source/file" true] ["bad.tmpfile" false]] -> root -> path
+  filterPattern = patterns: root:
+    (name: _type:
+      let
+        relPath = lib.removePrefix ((toString root) + "/") name;
+        matches = pair: (match (head pair) relPath) != null;
+        matched = map (pair: [(matches pair) (last pair)]) patterns;
+      in
+        last (last ([[true true]] ++ (filter head matched)))
+    );
+
+  # string -> [[regex bool]]
+  gitignoreToPatterns = gitignore:
+    assert throwIfOldNix;
+    let
+      # ignore -> bool
+      isComment = i: (match "^(#.*|$)" i) != null;
+
+      # ignore -> [ignore bool]
+      computeNegation = l:
+        let split = match "^(!?)(.*)" l;
+        in [(elemAt split 1) (head split == "!")];
+
+      # ignore -> regex
+      substWildcards =
+        let
+          special = "^$.+{}()";
+          escs = "\\*?";
+          splitString =
+            let recurse = str : [(substring 0 1 str)] ++
+                                 (if str == "" then [] else (recurse (substring 1 (stringLength(str)) str) ));
+            in str : recurse str;
+          chars = s: filter (c: c != "" && !isList c) (splitString s);
+          escape = s: map (c: "\\" + c) (chars s);
+        in
+          replaceStrings
+            ((chars special)  ++ (escape escs) ++ ["**/"    "**" "*"     "?"])
+            ((escape special) ++ (escape escs) ++ ["(.*/)?" ".*" "[^/]*" "[^/]"]);
+
+      # (regex -> regex) -> regex -> regex
+      mapAroundCharclass = f: r: # rl = regex or list
+        let slightFix = replaceStrings ["\\]"] ["]"];
+        in
+          concatStringsSep ""
+          (map (rl: if isList rl then slightFix (elemAt rl 0) else f rl)
+          (split "(\\[([^\\\\]|\\\\.)+])" r));
+
+      # regex -> regex
+      handleSlashPrefix = l:
+        let
+          split = (match "^(/?)(.*)" l);
+          findSlash = l: if (match ".+/.+" l) != null then "" else l;
+          hasSlash = mapAroundCharclass findSlash l != l;
+        in
+          (if (elemAt split 0) == "/" || hasSlash
+          then "^"
+          else "(^|.*/)"
+          ) + (elemAt split 1);
+
+      # regex -> regex
+      handleSlashSuffix = l:
+        let split = (match "^(.*)/$" l);
+        in if split != null then (elemAt split 0) + "($|/.*)" else l;
+
+      # (regex -> regex) -> [regex, bool] -> [regex, bool]
+      mapPat = f: l: [(f (head l)) (last l)];
+    in
+      map (l: # `l' for "line"
+        mapPat (l: handleSlashSuffix (handleSlashPrefix (mapAroundCharclass substWildcards l)))
+        (computeNegation l))
+      (filter (l: !isList l && !isComment l)
+      (split "\n" gitignore));
+
+  gitignoreFilter = ign: root: filterPattern (gitignoreToPatterns ign) root;
+
+  # string|[string|file] (→ [string|file] → [string]) -> string
+  gitignoreCompileIgnore = file_str_patterns: root:
+    let
+      onPath = f: a: if typeOf a == "path" then f a else a;
+      str_patterns = map (onPath readFile) (lib.toList file_str_patterns);
+    in concatStringsSep "\n" str_patterns;
+
+  gitignoreFilterPure = filter: patterns: root: name: type:
+    gitignoreFilter (gitignoreCompileIgnore patterns root) root name type
+    && filter name type;
+
+  # This is a very hacky way of programming this!
+  # A better way would be to reuse existing filtering by making multiple gitignore functions per each root.
+  # Then for each file find the set of roots with gitignores (and functions).
+  # This would make gitignoreFilterSource very different from gitignoreFilterPure.
+  # rootPath → gitignoresConcatenated
+  compileRecursiveGitignore = root:
+    let
+      dirOrIgnore = file: type: baseNameOf file == ".gitignore" || type == "directory";
+      ignores = builtins.filterSource dirOrIgnore root;
+    in readFile (
+      runCommand "${baseNameOf root}-recursive-gitignore" {} ''
+        cd ${ignores}
+
+        find -type f -exec sh -c '
+          rel="$(realpath --relative-to=. "$(dirname "$1")")/"
+          if [ "$rel" = "./" ]; then rel=""; fi
+
+          awk -v prefix="$rel" -v root="$1" -v top="$(test -z "$rel" && echo 1)" "
+            BEGIN { print \"# \"root }
+
+            /^!?[^\\/]+\/?$/ {
+              match(\$0, /^!?/, negation)
+              sub(/^!?/, \"\")
+
+              if (top) { middle = \"\" } else { middle = \"**/\" }
+
+              print negation[0] prefix middle \$0
+            }
+
+            /^!?(\\/|.*\\/.+$)/ {
+              match(\$0, /^!?/, negation)
+              sub(/^!?/, \"\")
+
+              if (!top) sub(/^\//, \"\")
+
+              print negation[0] prefix \$0
+            }
+
+            END { print \"\" }
+          " "$1"
+        ' sh {} \; > $out
+      '');
+
+  withGitignoreFile = patterns: root:
+    lib.toList patterns ++ [(root + "/.gitignore")];
+
+  withRecursiveGitignoreFile = patterns: root:
+    lib.toList patterns ++ [(compileRecursiveGitignore root)];
+
+  # filterSource derivatives
+
+  gitignoreFilterSourcePure = filter: patterns: root:
+    filterSource (gitignoreFilterPure filter patterns root) root;
+
+  gitignoreFilterSource = filter: patterns: root:
+    gitignoreFilterSourcePure filter (withGitignoreFile patterns root) root;
+
+  gitignoreFilterRecursiveSource = filter: patterns: root:
+    gitignoreFilterSourcePure filter (withRecursiveGitignoreFile patterns root) root;
+
+  # "Filter"-less alternatives
+
+  gitignoreSourcePure = gitignoreFilterSourcePure (_: _: true);
+  gitignoreSource = patterns: let type = typeOf patterns; in
+    if (type == "string" && pathExists patterns) || type == "path"
+    then throw
+      "type error in gitignoreSource(patterns -> source -> path), "
+      "use [] or \"\" if there are no additional patterns"
+    else gitignoreFilterSource (_: _: true) patterns;
+
+  gitignoreRecursiveSource = gitignoreFilterSourcePure (_: _: true);
+}

pkgs/top-level/all-packages.nix

@@ -309,6 +309,8 @@ in
 
   nixBufferBuilders = import ../build-support/emacs/buffer.nix { inherit (pkgs) lib writeText; inherit (emacsPackagesNg) inherit-local; };
 
+  nix-gitignore = callPackage ../build-support/nix-gitignore { };
+
   pathsFromGraph = ../build-support/kernel/paths-from-graph.pl;
 
   pruneLibtoolFiles = makeSetupHook { name = "prune-libtool-files"; }