nixos/documentation: split options doc build

most modules can be evaluated for their documentation in a very
restricted environment that doesn't include all of nixpkgs. this
evaluation can then be cached and reused for subsequent builds, merging
only documentation that has changed into the cached set. since nixos
ships with a large number of modules of which only a few are used in any
given config this can save evaluation a huge percentage of nixos
options available in any given config.

in tests of this caching, despite having to copy most of nixos/, saves
about 80% of the time needed to build the system manual, or about two
second on the machine used for testing. build time for a full system
config shrank from 9.4s to 7.4s, while turning documentation off
entirely shortened the build to 7.1s.
This commit is contained in:
pennae 2021-11-19 00:26:27 +01:00
parent 55daffc1c9
commit fc614c37c6
36 changed files with 384 additions and 22 deletions

View File

@ -177,7 +177,7 @@ rec {
docOption = rec {
loc = opt.loc;
name = showOption opt.loc;
description = opt.description or (lib.warn "Option `${name}' has no description." "This option has no description.");
description = opt.description or null;
declarations = filter (x: x != unknownModule) opt.declarations;
internal = opt.internal or false;
visible =

View File

@ -1,4 +1,4 @@
{ pkgs, options, config, version, revision, extraSources ? [] }:
{ pkgs, options, config, version, revision, extraSources ? [], baseOptionsJSON ? null, prefix ? ../../.. }:
with pkgs;
@ -11,11 +11,11 @@ let
#
# E.g. if some `options` came from modules in ${pkgs.customModules}/nix,
# you'd need to include `extraSources = [ pkgs.customModules ]`
prefixesToStrip = map (p: "${toString p}/") ([ ../../.. ] ++ extraSources);
prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources);
stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;
optionsDoc = buildPackages.nixosOptionsDoc {
inherit options revision;
inherit options revision baseOptionsJSON;
transformOptions = opt: opt // {
# Clean up declaration sites to not refer to the NixOS source tree.
declarations = map stripAnyPrefixes opt.declarations;
@ -161,7 +161,7 @@ let
in rec {
inherit generatedSources;
inherit (optionsDoc) optionsJSON optionsDocBook;
inherit (optionsDoc) optionsJSON optionsNix optionsDocBook;
# Generate the NixOS manual.
manualHTML = runCommand "nixos-manual-html"

View File

@ -5,7 +5,7 @@ extra information. Module meta attributes are defined in the `meta.nix`
special module.
`meta` is a top level attribute like `options` and `config`. Available
meta-attributes are `maintainers` and `doc`.
meta-attributes are `maintainers`, `doc`, and `buildDocsInSandbox`.
Each of the meta-attributes must be defined at most once per module
file.
@ -24,6 +24,7 @@ file.
meta = {
maintainers = with lib.maintainers; [ ericsagnes ];
doc = ./default.xml;
buildDocsInSandbox = true;
};
}
```
@ -38,3 +39,28 @@ file.
```ShellSession
$ nix-build nixos/release.nix -A manual.x86_64-linux
```
- `buildDocsInSandbox` indicates whether the option documentation for the
module can be built in a derivation sandbox. This option is currently only
honored for modules shipped by nixpkgs. User modules and modules taken from
`NIXOS_EXTRA_MODULE_PATH` are always built outside of the sandbox, as has
been the case in previous releases.
Building NixOS option documentation in a sandbox allows caching of the built
documentation, which greatly decreases the amount of time needed to evaluate
a system configuration that has NixOS documentation enabled. The sandbox also
restricts which attributes may be referenced by documentation attributes
(such as option descriptions) to the `options` and `lib` module arguments and
the `pkgs.formats` attribute of the `pkgs` argument, `config` and the rest of
`pkgs` are disallowed and will cause doc build failures when used. This
restriction is necessary because we cannot reproduce the full nixpkgs
instantiation with configuration and overlays from a system configuration
inside the sandbox. The `options` argument only includes options of modules
that are also built inside the sandbox, referencing an option of a module
that isn't built in the sandbox is also forbidden.
The default is `true` and should usually not be changed; set it to `false`
only if the module requires access to `pkgs` in its documentation (e.g.
because it loads information from a linked package to build an option type)
or if its documentation depends on other modules that also aren't sandboxed
(e.g. by using types defined in the other module).

View File

@ -8,8 +8,8 @@
<para>
<literal>meta</literal> is a top level attribute like
<literal>options</literal> and <literal>config</literal>. Available
meta-attributes are <literal>maintainers</literal> and
<literal>doc</literal>.
meta-attributes are <literal>maintainers</literal>,
<literal>doc</literal>, and <literal>buildDocsInSandbox</literal>.
</para>
<para>
Each of the meta-attributes must be defined at most once per module
@ -29,6 +29,7 @@
meta = {
maintainers = with lib.maintainers; [ ericsagnes ];
doc = ./default.xml;
buildDocsInSandbox = true;
};
}
</programlisting>
@ -51,5 +52,44 @@
$ nix-build nixos/release.nix -A manual.x86_64-linux
</programlisting>
</listitem>
<listitem>
<para>
<literal>buildDocsInSandbox</literal> indicates whether the
option documentation for the module can be built in a derivation
sandbox. This option is currently only honored for modules
shipped by nixpkgs. User modules and modules taken from
<literal>NIXOS_EXTRA_MODULE_PATH</literal> are always built
outside of the sandbox, as has been the case in previous
releases.
</para>
<para>
Building NixOS option documentation in a sandbox allows caching
of the built documentation, which greatly decreases the amount
of time needed to evaluate a system configuration that has NixOS
documentation enabled. The sandbox also restricts which
attributes may be referenced by documentation attributes (such
as option descriptions) to the <literal>options</literal> and
<literal>lib</literal> module arguments and the
<literal>pkgs.formats</literal> attribute of the
<literal>pkgs</literal> argument, <literal>config</literal> and
the rest of <literal>pkgs</literal> are disallowed and will
cause doc build failures when used. This restriction is
necessary because we cannot reproduce the full nixpkgs
instantiation with configuration and overlays from a system
configuration inside the sandbox. The <literal>options</literal>
argument only includes options of modules that are also built
inside the sandbox, referencing an option of a module that isnt
built in the sandbox is also forbidden.
</para>
<para>
The default is <literal>true</literal> and should usually not be
changed; set it to <literal>false</literal> only if the module
requires access to <literal>pkgs</literal> in its documentation
(e.g. because it loads information from a linked package to
build an option type) or if its documentation depends on other
modules that also arent sandboxed (e.g. by using types defined
in the other module).
</para>
</listitem>
</itemizedlist>
</section>

View File

@ -0,0 +1,53 @@
{ libPath
, pkgsLibPath
, nixosPath
, modules
, stateVersion
, release
}:
let
lib = import libPath;
modulesPath = "${nixosPath}/modules";
# dummy pkgs set that contains no packages, only `pkgs.lib` from the full set.
# not having `pkgs.lib` causes all users of `pkgs.formats` to fail.
pkgs = import pkgsLibPath {
inherit lib;
pkgs = null;
};
utils = import "${nixosPath}/lib/utils.nix" {
inherit config lib;
pkgs = null;
};
# this is used both as a module and as specialArgs.
# as a module it sets the _module special values, as specialArgs it makes `config`
# unusable. this causes documentation attributes depending on `config` to fail.
config = {
_module.check = false;
_module.args = {};
system.stateVersion = stateVersion;
};
eval = lib.evalModules {
modules = (map (m: "${modulesPath}/${m}") modules) ++ [
config
];
specialArgs = {
inherit config pkgs utils;
};
};
docs = import "${nixosPath}/doc/manual" {
pkgs = pkgs // {
inherit lib;
# duplicate of the declaration in all-packages.nix
buildPackages.nixosOptionsDoc = attrs:
(import "${nixosPath}/lib/make-options-doc")
({ inherit pkgs lib; } // attrs);
};
config = config.config;
options = eval.options;
version = release;
revision = "release-${release}";
prefix = modulesPath;
};
in
docs.optionsNix

View File

@ -21,6 +21,10 @@
, options
, transformOptions ? lib.id # function for additional tranformations of the options
, revision ? "" # Specify revision for the options
# a set of options the docs we are generating will be merged into, as if by recursiveUpdate.
# used to split the options doc build into a static part (nixos/modules) and a dynamic part
# (non-nixos modules imported via configuration.nix, other module sources).
, baseOptionsJSON ? null
}:
let
@ -99,13 +103,23 @@ in rec {
optionsJSON = pkgs.runCommand "options.json"
{ meta.description = "List of NixOS options in JSON format";
buildInputs = [ pkgs.brotli ];
options = builtins.toFile "options.json"
(builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix));
}
''
# Export list of options in different format.
dst=$out/share/doc/nixos
mkdir -p $dst
cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json
${
if baseOptionsJSON == null
then "cp $options $dst/options.json"
else ''
${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \
${baseOptionsJSON} $options \
> $dst/options.json
''
}
brotli -9 < $dst/options.json > $dst/options.json.br

View File

@ -0,0 +1,71 @@
import collections
import json
import sys
class Key:
def __init__(self, path):
self.path = path
def __hash__(self):
result = 0
for id in self.path:
result ^= hash(id)
return result
def __eq__(self, other):
return type(self) is type(other) and self.path == other.path
Option = collections.namedtuple('Option', ['name', 'value'])
# pivot a dict of options keyed by their display name to a dict keyed by their path
def pivot(options):
result = dict()
for (name, opt) in options.items():
result[Key(opt['loc'])] = Option(name, opt)
return result
# pivot back to indexed-by-full-name
# like the docbook build we'll just fail if multiple options with differing locs
# render to the same option name.
def unpivot(options):
result = dict()
for (key, opt) in options.items():
if opt.name in result:
raise RuntimeError(
'multiple options with colliding ids found',
opt.name,
result[opt.name]['loc'],
opt.value['loc'],
)
result[opt.name] = opt.value
return result
options = pivot(json.load(open(sys.argv[1], 'r')))
overrides = pivot(json.load(open(sys.argv[2], 'r')))
# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
for (k, v) in options.items():
v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations']))
# merge both descriptions
for (k, v) in overrides.items():
cur = options.setdefault(k, v).value
for (ok, ov) in v.value.items():
if ok == 'declarations':
decls = cur[ok]
for d in ov:
if d not in decls:
decls += [d]
elif ok == "type":
# ignore types of placeholder options
if ov != "_unspecified" or cur[ok] == "_unspecified":
cur[ok] = ov
elif ov is not None or cur.get(ok, None) is None:
cur[ok] = ov
# check that every option has a description
# TODO: nixos-rebuild with flakes may hide the warning, maybe turn on -L by default for those?
for (k, v) in options.items():
if v.value.get('description', None) is None:
print(f"\x1b[1;31mwarning: option {v.name} has no description\x1b[0m", file=sys.stderr)
v.value['description'] = "This option has no description."
json.dump(unpivot(options), fp=sys.stdout)

View File

@ -101,4 +101,7 @@ in
environment.systemPackages = packages;
};
# uses relatedPackages
meta.buildDocsInSandbox = false;
}

View File

@ -40,4 +40,7 @@ in
};
services.xserver.displayManager.sessionCommands = "${fcitxPackage}/bin/fcitx";
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -80,4 +80,7 @@ in
ibusPackage
];
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -45,5 +45,7 @@ in
environment.etc."xdg/kime/config.yaml".text = replaceStrings [ "\\\\" ] [ "\\" ] (builtins.toJSON cfg.config);
};
}
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -1,19 +1,35 @@
{ config, lib, pkgs, extendModules, noUserModules, ... }:
{ config, options, lib, pkgs, utils, modules, baseModules, extraModules, modulesPath, ... }:
with lib;
let
cfg = config.documentation;
allOpts = options;
/* Modules for which to show options even when not imported. */
extraDocModules = [ ../virtualisation/qemu-vm.nix ];
/* For the purpose of generating docs, evaluate options with each derivation
in `pkgs` (recursively) replaced by a fake with path "\${pkgs.attribute.path}".
It isn't perfect, but it seems to cover a vast majority of use cases.
Caveat: even if the package is reached by a different means,
the path above will be shown and not e.g. `${config.services.foo.package}`. */
canCacheDocs = m:
let
f = import m;
instance = f (mapAttrs (n: _: abort "evaluating ${n} for `meta` failed") (functionArgs f));
in
cfg.nixos.splitOptionDocBuild
&& builtins.isPath m
&& isFunction f
&& instance ? options
&& instance.meta.buildDocsInSandbox or true;
docModules =
let
p = partition canCacheDocs (baseModules ++ extraDocModules);
in
{
lazy = p.right;
eager = p.wrong ++ optionals cfg.nixos.includeAllModules (extraModules ++ modules);
};
manual = import ../../doc/manual rec {
inherit pkgs config;
version = config.system.nixos.release;
@ -21,10 +37,17 @@ let
extraSources = cfg.nixos.extraModuleSources;
options =
let
extendNixOS = if cfg.nixos.includeAllModules then extendModules else noUserModules.extendModules;
scrubbedEval = extendNixOS {
modules = extraDocModules;
specialArgs.pkgs = scrubDerivations "pkgs" pkgs;
scrubbedEval = evalModules {
modules = [ {
_module.check = false;
} ] ++ docModules.eager;
specialArgs = {
pkgs = scrubDerivations "pkgs" pkgs;
# allow access to arbitrary options for eager modules, eg for getting
# option types from lazy modules
options = allOpts;
inherit modulesPath utils;
};
};
scrubDerivations = namePrefix: pkgSet: mapAttrs
(name: value:
@ -36,6 +59,48 @@ let
)
pkgSet;
in scrubbedEval.options;
baseOptionsJSON =
let
filter =
builtins.filterSource
(n: t:
(t == "directory" -> baseNameOf n != "tests")
&& (t == "file" -> hasSuffix ".nix" n)
);
in
pkgs.runCommand "lazy-options.json" {
libPath = filter "${toString pkgs.path}/lib";
pkgsLibPath = filter "${toString pkgs.path}/pkgs/pkgs-lib";
nixosPath = filter "${toString pkgs.path}/nixos";
modules = map (p: ''"${removePrefix "${modulesPath}/" (toString p)}"'') docModules.lazy;
} ''
export NIX_STORE_DIR=$TMPDIR/store
export NIX_STATE_DIR=$TMPDIR/state
${pkgs.nix}/bin/nix-instantiate \
--show-trace \
--eval --json --strict \
--argstr libPath "$libPath" \
--argstr pkgsLibPath "$pkgsLibPath" \
--argstr nixosPath "$nixosPath" \
--arg modules "[ $modules ]" \
--argstr stateVersion "${options.system.stateVersion.default}" \
--argstr release "${config.system.nixos.release}" \
$nixosPath/lib/eval-cacheable-options.nix > $out \
|| {
echo -en "\e[1;31m"
echo 'Cacheable portion of option doc build failed.'
echo 'Usually this means that an option attribute that ends up in documentation (eg' \
'`default` or `description`) depends on the restricted module arguments' \
'`config` or `pkgs`.'
echo
echo 'Rebuild your configuration with `--show-trace` to find the offending' \
'location. Remove the references to restricted arguments (eg by escaping' \
'their antiquotations or adding a `defaultText`) or disable the sandboxed' \
'build for the failing module by setting `meta.buildDocsInSandbox = false`.'
echo -en "\e[0m"
exit 1
} >&2
'';
};
@ -191,6 +256,16 @@ in
'';
};
nixos.splitOptionDocBuild = mkOption {
type = types.bool;
default = true;
description = ''
Whether to split the option docs build into a cacheable and an uncacheable part.
Splitting the build can substantially decrease the amount of time needed to build
the manual, but some user modules may be incompatible with this splitting.
'';
};
nixos.includeAllModules = mkOption {
type = types.bool;
default = false;

View File

@ -54,6 +54,21 @@ in
'';
};
buildDocsInSandbox = mkOption {
type = types.bool // {
merge = loc: defs: defs;
};
internal = true;
default = true;
description = ''
Whether to include this module in the split options doc build.
Disable if the module references `config`, `pkgs` or other module
arguments that cannot be evaluated as constants.
This option should be defined at most once per module.
'';
};
};
};

View File

@ -248,4 +248,7 @@ in
)
];
};
# needs a full nixpkgs path to import nixpkgs
meta.buildDocsInSandbox = false;
}

View File

@ -119,4 +119,6 @@ in
};
# uses version info nixpkgs, which requires a full nixpkgs path
meta.buildDocsInSandbox = false;
}

View File

@ -7,6 +7,8 @@ let
in {
meta.maintainers = [ maintainers.etu ];
# uses relatedPackages
meta.buildDocsInSandbox = false;
###### interface
options = {

View File

@ -149,4 +149,6 @@ in
];
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -185,4 +185,7 @@ in {
imports = [
(lib.mkRenamedOptionModule [ "programs" "tmux" "extraTmuxConf" ] [ "programs" "tmux" "extraConfig" ])
];
# uses relatedPackages
meta.buildDocsInSandbox = false;
}

View File

@ -51,7 +51,10 @@ let
datasetOptions = rec {
use_template = mkOption {
description = "Names of the templates to use for this dataset.";
type = types.listOf (types.enum (attrNames cfg.templates));
type = types.listOf (types.str // {
check = (types.enum (attrNames cfg.templates)).check;
description = "configured template name";
});
default = [ ];
};
useTemplate = use_template;

View File

@ -29,6 +29,8 @@ in {
meta = {
maintainers = teams.freedesktop.members;
# uses attributes of the linked package
buildDocsInSandbox = false;
};
###### interface

View File

@ -40,6 +40,8 @@ in {
meta = {
maintainers = teams.freedesktop.members;
# uses attributes of the linked package
buildDocsInSandbox = false;
};
###### interface

View File

@ -221,4 +221,7 @@ in {
boot.extraModprobeConfig = "options thinkpad_acpi experimental=1 fan_control=1";
};
# uses relatedPackages
meta.buildDocsInSandbox = false;
}

View File

@ -226,4 +226,7 @@ in {
isSystemUser = true;
};
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -118,4 +118,7 @@ in
};
};
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -378,4 +378,6 @@ in
]);
meta.maintainers = with maintainers; [ hexa ];
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -228,5 +228,6 @@ in
};
meta.maintainers = with maintainers; [ rnhmjoj ];
# uses relatedPackages
meta.buildDocsInSandbox = false;
}

View File

@ -179,4 +179,7 @@ in {
wantedBy = [ "multi-user.target" ];
};
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -112,4 +112,7 @@ in
};
};
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -237,4 +237,6 @@ in
};
meta.maintainers = with lib.maintainers; [ edef zimbatm ];
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -167,4 +167,7 @@ in
"d ${cfg.dataDir}/async/ 0750 ${user} ${group} - -"
];
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -932,4 +932,6 @@ in {
]);
meta.doc = ./nextcloud.xml;
# uses relatedPackages
meta.buildDocsInSandbox = false;
}

View File

@ -146,4 +146,7 @@ in
group = "powerdnsadmin";
};
};
# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}

View File

@ -865,4 +865,6 @@ in
};
# uses relatedPackages
meta.buildDocsInSandbox = false;
}

View File

@ -317,4 +317,6 @@ in
};
# uses extendModules to generate a type
meta.buildDocsInSandbox = false;
}

View File

@ -999,4 +999,7 @@ in
];
};
# uses types of services/x11/xserver.nix
meta.buildDocsInSandbox = false;
}

View File

@ -451,4 +451,7 @@ in
};
# uses relatedPackages
meta.buildDocsInSandbox = false;
}