Document each store type on its own page

This makes for more useful manual table of contents, that displays the
information at a glance.

The `nix help-stores` command is kept as-is, even though it will show up
in the manual with the same information as these pages due to the way it
is written as a "`--help`-style" command. Deciding what to do with that
command is left for a later PR.

This change also lists all store types at the top of the respective overview page.

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems

.gitignore

@@ -22,6 +22,8 @@ perl/Makefile.config
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/SUMMARY-rl-next.md
+/doc/manual/src/store/types/*
+!/doc/manual/src/store/types/index.md.in
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md

doc/manual/generate-manpage.nix

@@ -51,7 +51,7 @@ let
 
         ${maybeSubcommands}
 
-        ${maybeStoreDocs}
+        ${maybeProse}
 
         ${maybeOptions}
       '';
@@ -91,25 +91,55 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      # FIXME: this is a hack.
-      # store parameters should not be part of command documentation to begin
-      # with, but instead be rendered on separate pages.
-      maybeStoreDocs = optionalString (details ? doc)
-        (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc);
+      maybeProse =
+        # FIXME: this is a horrible hack to keep `nix help-stores` working.
+        # the correct answer to this is to remove that command and replace it
+        # by statically generated manpages or the output of something like `nix
+        # store info <store type>`.
+        let
+          help-stores = ''
+            ${index}
 
-      maybeOptions = let
-        allVisibleOptions = filterAttrs
-          (_: o: ! o.hiddenCategory)
-          (details.flags // toplevel.flags);
-      in optionalString (allVisibleOptions != {}) ''
-        # Options
+            ${allStores}
+          '';
+          index = replaceStrings
+            [ "@store-types@" ] [ storesOverview ]
+            details.doc;
+          storesOverview =
+            let
+              showEntry = store:
+                "- [${store.name}](#${store.slug})";
+            in
+            concatStringsSep "\n" (map showEntry storesList) + "\n";
+          allStores = concatStringsSep "\n" (attrValues storePages);
+          storePages = listToAttrs
+            (map (s: { name = s.filename; value = s.page; }) storesList);
+          storesList = showStoreDocs {
+            storeInfo = commandInfo.stores;
+            inherit inlineHTML;
+          };
+        in
+        optionalString (details ? doc) (
+          if match "@store-types@" details.doc != [ ]
+          then help-stores
+          else details.doc
+        );
 
-        ${showOptions inlineHTML allVisibleOptions}
+      maybeOptions =
+        let
+          allVisibleOptions = filterAttrs
+            (_: o: ! o.hiddenCategory)
+            (details.flags // toplevel.flags);
+        in
+        optionalString (allVisibleOptions != { }) ''
+          # Options
 
-        > **Note**
-        >
-        > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
-      '';
+          ${showOptions inlineHTML allVisibleOptions}
+
+          > **Note**
+          >
+          > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
+        '';
 
       showOptions = inlineHTML: allOptions:
         let

doc/manual/generate-store-info.nix

@@ -1,14 +1,20 @@
 let
-  inherit (builtins) attrValues mapAttrs;
-  inherit (import <nix/utils.nix>) concatStrings optionalString squash;
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  inherit (import <nix/utils.nix>) optionalString filterAttrs trim squash toLower unique indent;
   showSettings = import <nix/generate-settings.nix>;
 in
 
-inlineHTML: storesInfo:
+{
+  # data structure describing all stores and their parameters
+  storeInfo,
+  # whether to add inline HTML tags
+  # `lowdown` does not eat those for one of the output modes
+  inlineHTML,
+}:
 
 let
 
-  showStore = name: { settings, doc, experimentalFeature }:
+  showStore = { name, slug }: { settings, doc, experimentalFeature }:
     let
       result = squash ''
         # ${name}
@@ -22,9 +28,6 @@ let
         ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
       '';
 
-      # markdown doesn't like spaces in URLs
-      slug = builtins.replaceStrings [ " " ] [ "-" ] name;
-
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
         > **Warning**
         >
@@ -42,4 +45,13 @@ let
       '';
     in result;
 
-in concatStrings (attrValues (mapAttrs showStore storesInfo))
+  storesList = map
+    (name: rec {
+      inherit name;
+      slug = replaceStrings [ " " ] [ "-" ] (toLower name);
+      filename = "${slug}.md";
+      page = showStore { inherit name slug; } storeInfo.${name};
+    })
+    (attrNames storeInfo);
+
+in storesList

doc/manual/generate-store-types.nix

@@ -0,0 +1,39 @@
+let
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  showSettings = import <nix/generate-settings.nix>;
+  showStoreDocs = import <nix/generate-store-info.nix>;
+in
+
+storeInfo:
+
+let
+  storesList = showStoreDocs {
+    inherit storeInfo;
+    inlineHTML = true;
+  };
+
+  index =
+    let
+      showEntry = store:
+        "- [${store.name}](./${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList);
+
+  "index.md" = replaceStrings
+    [ "@store-types@" ] [ index ]
+    (readFile ./src/store/types/index.md.in);
+
+  tableOfContents =
+    let
+      showEntry = store:
+        "    - [${store.name}](store/types/${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList) + "\n";
+
+  "SUMMARY.md" = tableOfContents;
+
+  storePages = listToAttrs
+    (map (s: { name = s.filename; value = s.page; }) storesList);
+
+in
+storePages // { inherit "index.md" "SUMMARY.md"; }

doc/manual/generate-xp-features.nix

@@ -8,4 +8,6 @@ let
 
       ${doc}
     '';
-in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))
+in
+
+xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/local.mk

@@ -97,10 +97,17 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
+$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix)
+	@# FIXME: build out of tree!
+	@rm -rf $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores'
+	@# do not destroy existing contents
+	@mv $@.tmp/* $@/
+
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
@@ -200,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/src/SUMMARY.md.in

@@ -20,6 +20,8 @@
   - [File System Object](store/file-system-object.md)
   - [Store Object](store/store-object.md)
   - [Store Path](store/store-path.md)
+  - [Store Types](store/types/index.md)
+{{#include ./store/types/SUMMARY.md}}
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)

doc/manual/src/contributing/cli-guideline.md

@@ -426,7 +426,7 @@ This leads to the following guidelines:
 ### Examples
 
 
-This is bad, because all keys must be assumed to be store implementations:
+This is bad, because all keys must be assumed to be store types:
 
 ```json
 {

doc/manual/src/store/index.md

@@ -2,4 +2,4 @@
 
 The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
 
-There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.
+There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md).

doc/manual/src/store/types/index.md.in

@@ -1,4 +1,6 @@
-Nix supports different types of stores. These are described below.
+Nix supports different types of stores:
+
+@store-types@
 
 ## Store URL format
 
@@ -39,4 +41,3 @@ store as follows:
 
 * Otherwise, use the [local store](#local-store) `/nix/store`.
 
-@stores@

doc/manual/utils.nix

@@ -1,5 +1,11 @@
 with builtins;
 
+let
+  lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz";
+  upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  stringToCharacters = s: genList (p: substring p 1 s) (stringLength s);
+in
+
 rec {
   splitLines = s: filter (x: !isList x) (split "\n" s);
 
@@ -18,6 +24,8 @@ rec {
     in
       if replaced == string then string else replaceStringsRec from to replaced;
 
+  toLower = replaceStrings upperChars lowerChars;
+
   squash = replaceStringsRec "\n\n\n" "\n\n";
 
   trim = string:

src/libcmd/common-eval-args.cc

@@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs()
         .longName = "eval-store",
         .description =
           R"(
-            The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+            The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
             to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
           )",
         .category = category,

src/libexpr/primops.cc

@@ -4460,7 +4460,7 @@ void EvalState::createBaseEnv()
         .doc = R"(
           Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
 
-          This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
+          This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format):
 
           ```shell-session
           $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir

src/libstore/globals.hh

@@ -117,10 +117,11 @@ public:
 
     Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
         R"(
-          The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
           to use for most operations.
-          See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
-          for supported store types and settings.
+          See the
+          [Store Types](@docroot@/store/types/index.md)
+          section of the manual for supported store types and settings.
         )"};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
@@ -759,7 +760,7 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
+          A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
           A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
           Substituters are tried based on their priority value, which each substituter can set independently.
@@ -778,7 +779,7 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
+          A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace.
           These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters).
 
           Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.

src/libstore/remote-store.cc

@@ -20,7 +20,7 @@
 
 namespace nix {
 
-/* TODO: Separate these store impls into different files, give them better names */
+/* TODO: Separate these store types into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : RemoteStoreConfig(params)
     , Store(params)

src/libstore/ssh-store-config.hh

@@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
 
     const Setting<std::string> remoteStore{this, "", "remote-store",
         R"(
-          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          [Store URL](@docroot@/store/types/index.md#store-url-format)
           to be used on the remote machine. The default is `auto`
           (i.e. use the Nix daemon or `/nix/store` directly).
         )"};

src/libstore/store-api.hh

@@ -30,7 +30,7 @@
 namespace nix {
 
 /**
- * About the class hierarchy of the store implementations:
+ * About the class hierarchy of the store types:
  *
  * Each store type `Foo` consists of two classes:
  *
@@ -962,7 +962,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
  * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
  *   ‘nix-store --serve’ via SSH.
  *
- * You can pass parameters to the store implementation by appending
+ * You can pass parameters to the store type by appending
  * ‘?key=value&key=value&...’ to the URI.
  */
 ref<Store> openStore(const std::string & uri = settings.storeUri.get(),

src/libutil/experimental-features.cc

@@ -252,7 +252,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .tag = Xp::ReadOnlyLocalStore,
         .name = "read-only-local-store",
         .description = R"(
-            Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs.
+            Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs.
         )",
     },
     {

src/nix/nix.md

@@ -235,10 +235,14 @@ operate are determined as follows:
 
 # Nix stores
 
-Most `nix` subcommands operate on a *Nix store*. These are documented
-in [`nix help-stores`](./nix3-help-stores.md).
+Most `nix` subcommands operate on a *Nix store*.
+The various store types are documented in the
+[Store Types](@docroot@/store/types/index.md)
+section of the manual.
 
-# Shebang interpreter 
+The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command.
+
+# Shebang interpreter
 
 The `nix` command can be used as a `#!` interpreter.
 Arguments to Nix can be passed on subsequent lines in the script.