From abef4b10b6d75da66025b9f9fe095e820f8b96ad Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 8 Dec 2021 05:08:23 +0100 Subject: [PATCH 01/11] nixos/kubernetes: add missing defaultText to expression default --- nixos/modules/services/cluster/kubernetes/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index cf7fcb0a6d73..af39b87fa772 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -198,6 +198,9 @@ in { description = "Default location for kubernetes secrets. Not a store location."; type = types.path; default = cfg.dataDir + "/secrets"; + defaultText = literalExpression '' + config.${opt.dataDir} + "/secrets" + ''; }; }; From 7e28421e1704c95c056f2b2e7fc27a7569182e0f Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 8 Dec 2021 05:09:32 +0100 Subject: [PATCH 02/11] nixos/kubernetes: make lib option internal and readonly this set almost certainly shouldn't be touched by users, nor listed in the manual. make it internal and use it only through the option path to make clear that this should not be modified. --- .../services/cluster/kubernetes/controller-manager.nix | 7 ++++--- nixos/modules/services/cluster/kubernetes/default.nix | 2 ++ nixos/modules/services/cluster/kubernetes/kubelet.nix | 7 ++++--- nixos/modules/services/cluster/kubernetes/pki.nix | 9 +++++---- nixos/modules/services/cluster/kubernetes/proxy.nix | 7 ++++--- nixos/modules/services/cluster/kubernetes/scheduler.nix | 7 ++++--- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix index ed25715fab7d..6d54659720cb 100644 --- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix +++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix @@ -6,6 +6,7 @@ let top = config.services.kubernetes; otop = options.services.kubernetes; cfg = top.controllerManager; + klib = options.services.kubernetes.lib.default; in { imports = [ @@ -56,7 +57,7 @@ in type = int; }; - kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager"; + kubeconfig = klib.mkKubeConfigOptions "Kubernetes controller manager"; leaderElect = mkOption { description = "Whether to start leader election before executing main loop."; @@ -129,7 +130,7 @@ in "--cluster-cidr=${cfg.clusterCidr}"} \ ${optionalString (cfg.featureGates != []) "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ - --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \ + --kubeconfig=${klib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \ --leader-elect=${boolToString cfg.leaderElect} \ ${optionalString (cfg.rootCaFile!=null) "--root-ca-file=${cfg.rootCaFile}"} \ @@ -156,7 +157,7 @@ in path = top.path; }; - services.kubernetes.pki.certs = with top.lib; { + services.kubernetes.pki.certs = with klib; { controllerManager = mkCert { name = "kube-controller-manager"; CN = "kube-controller-manager"; diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index af39b87fa772..807d8d1a193d 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -192,6 +192,8 @@ in { inherit mkKubeConfigOptions; }; type = types.attrs; + readOnly = true; + internal = true; }; secretsPath = mkOption { diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix index 3e8eac96f6ba..2d58547ce4ce 100644 --- a/nixos/modules/services/cluster/kubernetes/kubelet.nix +++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix @@ -6,6 +6,7 @@ let top = config.services.kubernetes; otop = options.services.kubernetes; cfg = top.kubelet; + klib = options.services.kubernetes.lib.default; cniConfig = if cfg.cni.config != [] && cfg.cni.configDir != null then @@ -27,7 +28,7 @@ let config.Cmd = ["/bin/pause"]; }; - kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig; + kubeconfig = klib.mkKubeConfig "kubelet" cfg.kubeconfig; manifestPath = "kubernetes/manifests"; @@ -177,7 +178,7 @@ in type = str; }; - kubeconfig = top.lib.mkKubeConfigOptions "Kubelet"; + kubeconfig = klib.mkKubeConfigOptions "Kubelet"; manifests = mkOption { description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; @@ -358,7 +359,7 @@ in services.kubernetes.kubelet.hostname = with config.networking; mkDefault (hostName + optionalString (domain != null) ".${domain}"); - services.kubernetes.pki.certs = with top.lib; { + services.kubernetes.pki.certs = with klib; { kubelet = mkCert { name = "kubelet"; CN = top.kubelet.hostname; diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 76ab03cd520b..00d572a50988 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -1,10 +1,11 @@ -{ config, lib, pkgs, ... }: +{ config, options, lib, pkgs, ... }: with lib; let top = config.services.kubernetes; cfg = top.pki; + klib = options.services.kubernetes.lib; csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON { key = { @@ -29,7 +30,7 @@ let cfsslAPITokenLength = 32; clusterAdminKubeconfig = with cfg.certs.clusterAdmin; - top.lib.mkKubeConfig "cluster-admin" { + klib.mkKubeConfig "cluster-admin" { server = top.apiserverAddress; certFile = cert; keyFile = key; @@ -250,7 +251,7 @@ in # - it would be better with a more Nix-oriented way of managing addons systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ environment.KUBECONFIG = with cfg.certs.addonManager; - top.lib.mkKubeConfig "addon-manager" { + klib.mkKubeConfig "addon-manager" { server = top.apiserverAddress; certFile = cert; keyFile = key; @@ -343,7 +344,7 @@ in ''; services.flannel = with cfg.certs.flannelClient; { - kubeconfig = top.lib.mkKubeConfig "flannel" { + kubeconfig = klib.mkKubeConfig "flannel" { server = top.apiserverAddress; certFile = cert; keyFile = key; diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix index 5f3da034120b..986301f6bd95 100644 --- a/nixos/modules/services/cluster/kubernetes/proxy.nix +++ b/nixos/modules/services/cluster/kubernetes/proxy.nix @@ -6,6 +6,7 @@ let top = config.services.kubernetes; otop = options.services.kubernetes; cfg = top.proxy; + klib = options.services.kubernetes.lib.default; in { imports = [ @@ -43,7 +44,7 @@ in type = str; }; - kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes proxy"; + kubeconfig = klib.mkKubeConfigOptions "Kubernetes proxy"; verbosity = mkOption { description = '' @@ -72,7 +73,7 @@ in ${optionalString (cfg.featureGates != []) "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ --hostname-override=${cfg.hostname} \ - --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \ + --kubeconfig=${klib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ ${cfg.extraOpts} ''; @@ -88,7 +89,7 @@ in services.kubernetes.proxy.hostname = with config.networking; mkDefault hostName; services.kubernetes.pki.certs = { - kubeProxyClient = top.lib.mkCert { + kubeProxyClient = klib.mkCert { name = "kube-proxy-client"; CN = "system:kube-proxy"; action = "systemctl restart kube-proxy.service"; diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix index 87263ee72fa4..442e3fe3a69f 100644 --- a/nixos/modules/services/cluster/kubernetes/scheduler.nix +++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix @@ -6,6 +6,7 @@ let top = config.services.kubernetes; otop = options.services.kubernetes; cfg = top.scheduler; + klib = options.services.kubernetes.lib.default; in { ###### interface @@ -32,7 +33,7 @@ in type = listOf str; }; - kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler"; + kubeconfig = klib.mkKubeConfigOptions "Kubernetes scheduler"; leaderElect = mkOption { description = "Whether to start leader election before executing main loop."; @@ -69,7 +70,7 @@ in --address=${cfg.address} \ ${optionalString (cfg.featureGates != []) "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ - --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \ + --kubeconfig=${klib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \ --leader-elect=${boolToString cfg.leaderElect} \ --port=${toString cfg.port} \ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ @@ -87,7 +88,7 @@ in }; services.kubernetes.pki.certs = { - schedulerClient = top.lib.mkCert { + schedulerClient = klib.mkCert { name = "kube-scheduler-client"; CN = "system:kube-scheduler"; action = "systemctl restart kube-scheduler.service"; From 55863f14ce1d4c4f5f3b961315dc8d94b832d12c Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 09:28:04 +0100 Subject: [PATCH 03/11] nixos/couchdb: add missing defaultText --- nixos/modules/services/databases/couchdb.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/databases/couchdb.nix b/nixos/modules/services/databases/couchdb.nix index 266bc82b6967..742e605d224d 100644 --- a/nixos/modules/services/databases/couchdb.nix +++ b/nixos/modules/services/databases/couchdb.nix @@ -1,9 +1,10 @@ -{ config, lib, pkgs, ... }: +{ config, options, lib, pkgs, ... }: with lib; let cfg = config.services.couchdb; + opt = options.services.couchdb; configFile = pkgs.writeText "couchdb.ini" ( '' [couchdb] @@ -153,6 +154,7 @@ in { argsFile = mkOption { type = types.path; default = "${cfg.package}/etc/vm.args"; + defaultText = literalExpression ''"config.${opt.package}/etc/vm.args"''; description = '' vm.args configuration. Overrides Couchdb's Erlang VM parameters file. ''; From bf58a90d09fdfb3f24b304be79e1259b123d8632 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 09:28:23 +0100 Subject: [PATCH 04/11] nixos/xrdp: add missing defaultText --- nixos/modules/services/networking/xrdp.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/modules/services/networking/xrdp.nix b/nixos/modules/services/networking/xrdp.nix index e9f123a181ae..747fb7a1f9c4 100644 --- a/nixos/modules/services/networking/xrdp.nix +++ b/nixos/modules/services/networking/xrdp.nix @@ -100,6 +100,7 @@ in confDir = mkOption { type = types.path; default = confDir; + defaultText = literalDocBook "generated from configuration"; description = "The location of the config files for xrdp."; }; }; From 3dbb117aa579330d76fd3b949862b1223f91d6ea Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 09:28:37 +0100 Subject: [PATCH 05/11] nixos/aesmd: add missing defaultText --- nixos/modules/services/security/aesmd.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/security/aesmd.nix b/nixos/modules/services/security/aesmd.nix index bb53bc49e259..924d614e4717 100644 --- a/nixos/modules/services/security/aesmd.nix +++ b/nixos/modules/services/security/aesmd.nix @@ -1,7 +1,8 @@ -{ config, pkgs, lib, ... }: +{ config, options, pkgs, lib, ... }: with lib; let cfg = config.services.aesmd; + opt = options.services.aesmd; sgx-psw = pkgs.sgx-psw.override { inherit (cfg) debug; }; @@ -43,6 +44,9 @@ in options.proxyType = mkOption { type = with types; nullOr (enum [ "default" "direct" "manual" ]); default = if (cfg.settings.proxy != null) then "manual" else null; + defaultText = literalExpression '' + if (config.${opt.settings}.proxy != null) then "manual" else null + ''; example = "default"; description = '' Type of proxy to use. The default uses the system's default proxy. From 55daffc1c943bddb71dc89a606f8284f6d50f5bd Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 29 Dec 2021 20:09:30 +0100 Subject: [PATCH 06/11] nixos/sourcehut: add missing defaultText, escape antiquotations --- nixos/modules/services/misc/sourcehut/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/misc/sourcehut/default.nix b/nixos/modules/services/misc/sourcehut/default.nix index 1bd21c278e00..21551d7d5f03 100644 --- a/nixos/modules/services/misc/sourcehut/default.nix +++ b/nixos/modules/services/misc/sourcehut/default.nix @@ -678,7 +678,7 @@ in rev = "ff96a0fa5635770390b184ae74debea75c3fd534"; ref = "nixos-unstable"; }; - image_from_nixpkgs = (import ("${pkgs.sourcehut.buildsrht}/lib/images/nixos/image.nix") { + image_from_nixpkgs = (import ("''${pkgs.sourcehut.buildsrht}/lib/images/nixos/image.nix") { pkgs = (import pkgs_unstable {}); }); in @@ -696,6 +696,7 @@ in package = mkOption { type = types.package; default = pkgs.git; + defaultText = literalExpression "pkgs.git"; example = literalExpression "pkgs.gitFull"; description = '' Git package for git.sr.ht. This can help silence collisions. @@ -712,6 +713,7 @@ in package = mkOption { type = types.package; default = pkgs.mercurial; + defaultText = literalExpression "pkgs.mercurial"; description = '' Mercurial package for hg.sr.ht. This can help silence collisions. ''; From fc614c37c653637e5475a0b0a987489b4d1f351d Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 19 Nov 2021 00:26:27 +0100 Subject: [PATCH 07/11] nixos/documentation: split options doc build most modules can be evaluated for their documentation in a very restricted environment that doesn't include all of nixpkgs. this evaluation can then be cached and reused for subsequent builds, merging only documentation that has changed into the cached set. since nixos ships with a large number of modules of which only a few are used in any given config this can save evaluation a huge percentage of nixos options available in any given config. in tests of this caching, despite having to copy most of nixos/, saves about 80% of the time needed to build the system manual, or about two second on the machine used for testing. build time for a full system config shrank from 9.4s to 7.4s, while turning documentation off entirely shortened the build to 7.1s. --- lib/options.nix | 2 +- nixos/doc/manual/default.nix | 8 +- .../development/meta-attributes.section.md | 28 +++++- .../development/meta-attributes.section.xml | 44 ++++++++- nixos/lib/eval-cacheable-options.nix | 53 +++++++++++ nixos/lib/make-options-doc/default.nix | 16 +++- nixos/lib/make-options-doc/mergeJSON.py | 71 ++++++++++++++ nixos/modules/config/qt5.nix | 3 + nixos/modules/i18n/input-method/fcitx.nix | 3 + nixos/modules/i18n/input-method/ibus.nix | 3 + nixos/modules/i18n/input-method/kime.nix | 4 +- nixos/modules/misc/documentation.nix | 95 +++++++++++++++++-- nixos/modules/misc/meta.nix | 15 +++ nixos/modules/misc/nixpkgs.nix | 3 + nixos/modules/misc/version.nix | 2 + nixos/modules/programs/dmrconfig.nix | 2 + nixos/modules/programs/gnupg.nix | 2 + nixos/modules/programs/tmux.nix | 3 + nixos/modules/services/backup/sanoid.nix | 5 +- .../pipewire/pipewire-media-session.nix | 2 + .../services/desktops/pipewire/pipewire.nix | 2 + nixos/modules/services/hardware/thinkfan.nix | 3 + .../services/misc/matrix-appservice-irc.nix | 3 + .../services/networking/dnscrypt-proxy2.nix | 3 + nixos/modules/services/networking/kea.nix | 2 + nixos/modules/services/networking/searx.nix | 3 +- .../services/security/vaultwarden/default.nix | 3 + nixos/modules/services/web-apps/dex.nix | 3 + nixos/modules/services/web-apps/gerrit.nix | 2 + nixos/modules/services/web-apps/jirafeau.nix | 3 + nixos/modules/services/web-apps/nextcloud.nix | 2 + .../services/web-apps/powerdns-admin.nix | 3 + nixos/modules/services/x11/xserver.nix | 2 + nixos/modules/system/activation/top-level.nix | 2 + nixos/modules/virtualisation/qemu-vm.nix | 3 + nixos/modules/virtualisation/xen-dom0.nix | 3 + 36 files changed, 384 insertions(+), 22 deletions(-) create mode 100644 nixos/lib/eval-cacheable-options.nix create mode 100644 nixos/lib/make-options-doc/mergeJSON.py diff --git a/lib/options.nix b/lib/options.nix index 5d52f065af08..53001a3113f9 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -177,7 +177,7 @@ rec { docOption = rec { loc = opt.loc; name = showOption opt.loc; - description = opt.description or (lib.warn "Option `${name}' has no description." "This option has no description."); + description = opt.description or null; declarations = filter (x: x != unknownModule) opt.declarations; internal = opt.internal or false; visible = diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 31b6da01c6bd..9bc63686fa3a 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -1,4 +1,4 @@ -{ pkgs, options, config, version, revision, extraSources ? [] }: +{ pkgs, options, config, version, revision, extraSources ? [], baseOptionsJSON ? null, prefix ? ../../.. }: with pkgs; @@ -11,11 +11,11 @@ let # # E.g. if some `options` came from modules in ${pkgs.customModules}/nix, # you'd need to include `extraSources = [ pkgs.customModules ]` - prefixesToStrip = map (p: "${toString p}/") ([ ../../.. ] ++ extraSources); + prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources); stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip; optionsDoc = buildPackages.nixosOptionsDoc { - inherit options revision; + inherit options revision baseOptionsJSON; transformOptions = opt: opt // { # Clean up declaration sites to not refer to the NixOS source tree. declarations = map stripAnyPrefixes opt.declarations; @@ -161,7 +161,7 @@ let in rec { inherit generatedSources; - inherit (optionsDoc) optionsJSON optionsDocBook; + inherit (optionsDoc) optionsJSON optionsNix optionsDocBook; # Generate the NixOS manual. manualHTML = runCommand "nixos-manual-html" diff --git a/nixos/doc/manual/development/meta-attributes.section.md b/nixos/doc/manual/development/meta-attributes.section.md index ca4ba007f7dc..946c08efd0a3 100644 --- a/nixos/doc/manual/development/meta-attributes.section.md +++ b/nixos/doc/manual/development/meta-attributes.section.md @@ -5,7 +5,7 @@ extra information. Module meta attributes are defined in the `meta.nix` special module. `meta` is a top level attribute like `options` and `config`. Available -meta-attributes are `maintainers` and `doc`. +meta-attributes are `maintainers`, `doc`, and `buildDocsInSandbox`. Each of the meta-attributes must be defined at most once per module file. @@ -24,6 +24,7 @@ file. meta = { maintainers = with lib.maintainers; [ ericsagnes ]; doc = ./default.xml; + buildDocsInSandbox = true; }; } ``` @@ -38,3 +39,28 @@ file. ```ShellSession $ nix-build nixos/release.nix -A manual.x86_64-linux ``` + +- `buildDocsInSandbox` indicates whether the option documentation for the + module can be built in a derivation sandbox. This option is currently only + honored for modules shipped by nixpkgs. User modules and modules taken from + `NIXOS_EXTRA_MODULE_PATH` are always built outside of the sandbox, as has + been the case in previous releases. + + Building NixOS option documentation in a sandbox allows caching of the built + documentation, which greatly decreases the amount of time needed to evaluate + a system configuration that has NixOS documentation enabled. The sandbox also + restricts which attributes may be referenced by documentation attributes + (such as option descriptions) to the `options` and `lib` module arguments and + the `pkgs.formats` attribute of the `pkgs` argument, `config` and the rest of + `pkgs` are disallowed and will cause doc build failures when used. This + restriction is necessary because we cannot reproduce the full nixpkgs + instantiation with configuration and overlays from a system configuration + inside the sandbox. The `options` argument only includes options of modules + that are also built inside the sandbox, referencing an option of a module + that isn't built in the sandbox is also forbidden. + + The default is `true` and should usually not be changed; set it to `false` + only if the module requires access to `pkgs` in its documentation (e.g. + because it loads information from a linked package to build an option type) + or if its documentation depends on other modules that also aren't sandboxed + (e.g. by using types defined in the other module). diff --git a/nixos/doc/manual/from_md/development/meta-attributes.section.xml b/nixos/doc/manual/from_md/development/meta-attributes.section.xml index f535d94602bd..1eb6e0f30368 100644 --- a/nixos/doc/manual/from_md/development/meta-attributes.section.xml +++ b/nixos/doc/manual/from_md/development/meta-attributes.section.xml @@ -8,8 +8,8 @@ meta is a top level attribute like options and config. Available - meta-attributes are maintainers and - doc. + meta-attributes are maintainers, + doc, and buildDocsInSandbox. Each of the meta-attributes must be defined at most once per module @@ -29,6 +29,7 @@ meta = { maintainers = with lib.maintainers; [ ericsagnes ]; doc = ./default.xml; + buildDocsInSandbox = true; }; } @@ -51,5 +52,44 @@ $ nix-build nixos/release.nix -A manual.x86_64-linux + + + buildDocsInSandbox indicates whether the + option documentation for the module can be built in a derivation + sandbox. This option is currently only honored for modules + shipped by nixpkgs. User modules and modules taken from + NIXOS_EXTRA_MODULE_PATH are always built + outside of the sandbox, as has been the case in previous + releases. + + + Building NixOS option documentation in a sandbox allows caching + of the built documentation, which greatly decreases the amount + of time needed to evaluate a system configuration that has NixOS + documentation enabled. The sandbox also restricts which + attributes may be referenced by documentation attributes (such + as option descriptions) to the options and + lib module arguments and the + pkgs.formats attribute of the + pkgs argument, config and + the rest of pkgs are disallowed and will + cause doc build failures when used. This restriction is + necessary because we cannot reproduce the full nixpkgs + instantiation with configuration and overlays from a system + configuration inside the sandbox. The options + argument only includes options of modules that are also built + inside the sandbox, referencing an option of a module that isn’t + built in the sandbox is also forbidden. + + + The default is true and should usually not be + changed; set it to false only if the module + requires access to pkgs in its documentation + (e.g. because it loads information from a linked package to + build an option type) or if its documentation depends on other + modules that also aren’t sandboxed (e.g. by using types defined + in the other module). + + diff --git a/nixos/lib/eval-cacheable-options.nix b/nixos/lib/eval-cacheable-options.nix new file mode 100644 index 000000000000..c3ba2ce66375 --- /dev/null +++ b/nixos/lib/eval-cacheable-options.nix @@ -0,0 +1,53 @@ +{ libPath +, pkgsLibPath +, nixosPath +, modules +, stateVersion +, release +}: + +let + lib = import libPath; + modulesPath = "${nixosPath}/modules"; + # dummy pkgs set that contains no packages, only `pkgs.lib` from the full set. + # not having `pkgs.lib` causes all users of `pkgs.formats` to fail. + pkgs = import pkgsLibPath { + inherit lib; + pkgs = null; + }; + utils = import "${nixosPath}/lib/utils.nix" { + inherit config lib; + pkgs = null; + }; + # this is used both as a module and as specialArgs. + # as a module it sets the _module special values, as specialArgs it makes `config` + # unusable. this causes documentation attributes depending on `config` to fail. + config = { + _module.check = false; + _module.args = {}; + system.stateVersion = stateVersion; + }; + eval = lib.evalModules { + modules = (map (m: "${modulesPath}/${m}") modules) ++ [ + config + ]; + specialArgs = { + inherit config pkgs utils; + }; + }; + docs = import "${nixosPath}/doc/manual" { + pkgs = pkgs // { + inherit lib; + # duplicate of the declaration in all-packages.nix + buildPackages.nixosOptionsDoc = attrs: + (import "${nixosPath}/lib/make-options-doc") + ({ inherit pkgs lib; } // attrs); + }; + config = config.config; + options = eval.options; + version = release; + revision = "release-${release}"; + prefix = modulesPath; + }; +in + docs.optionsNix diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 44bc25be9238..4b40af34b185 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -21,6 +21,10 @@ , options , transformOptions ? lib.id # function for additional tranformations of the options , revision ? "" # Specify revision for the options +# a set of options the docs we are generating will be merged into, as if by recursiveUpdate. +# used to split the options doc build into a static part (nixos/modules) and a dynamic part +# (non-nixos modules imported via configuration.nix, other module sources). +, baseOptionsJSON ? null }: let @@ -99,13 +103,23 @@ in rec { optionsJSON = pkgs.runCommand "options.json" { meta.description = "List of NixOS options in JSON format"; buildInputs = [ pkgs.brotli ]; + options = builtins.toFile "options.json" + (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix)); } '' # Export list of options in different format. dst=$out/share/doc/nixos mkdir -p $dst - cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json + ${ + if baseOptionsJSON == null + then "cp $options $dst/options.json" + else '' + ${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \ + ${baseOptionsJSON} $options \ + > $dst/options.json + '' + } brotli -9 < $dst/options.json > $dst/options.json.br diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py new file mode 100644 index 000000000000..e7f6897c6d0d --- /dev/null +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -0,0 +1,71 @@ +import collections +import json +import sys + +class Key: + def __init__(self, path): + self.path = path + def __hash__(self): + result = 0 + for id in self.path: + result ^= hash(id) + return result + def __eq__(self, other): + return type(self) is type(other) and self.path == other.path + +Option = collections.namedtuple('Option', ['name', 'value']) + +# pivot a dict of options keyed by their display name to a dict keyed by their path +def pivot(options): + result = dict() + for (name, opt) in options.items(): + result[Key(opt['loc'])] = Option(name, opt) + return result + +# pivot back to indexed-by-full-name +# like the docbook build we'll just fail if multiple options with differing locs +# render to the same option name. +def unpivot(options): + result = dict() + for (key, opt) in options.items(): + if opt.name in result: + raise RuntimeError( + 'multiple options with colliding ids found', + opt.name, + result[opt.name]['loc'], + opt.value['loc'], + ) + result[opt.name] = opt.value + return result + +options = pivot(json.load(open(sys.argv[1], 'r'))) +overrides = pivot(json.load(open(sys.argv[2], 'r'))) + +# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir +for (k, v) in options.items(): + v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations'])) + +# merge both descriptions +for (k, v) in overrides.items(): + cur = options.setdefault(k, v).value + for (ok, ov) in v.value.items(): + if ok == 'declarations': + decls = cur[ok] + for d in ov: + if d not in decls: + decls += [d] + elif ok == "type": + # ignore types of placeholder options + if ov != "_unspecified" or cur[ok] == "_unspecified": + cur[ok] = ov + elif ov is not None or cur.get(ok, None) is None: + cur[ok] = ov + +# check that every option has a description +# TODO: nixos-rebuild with flakes may hide the warning, maybe turn on -L by default for those? +for (k, v) in options.items(): + if v.value.get('description', None) is None: + print(f"\x1b[1;31mwarning: option {v.name} has no description\x1b[0m", file=sys.stderr) + v.value['description'] = "This option has no description." + +json.dump(unpivot(options), fp=sys.stdout) diff --git a/nixos/modules/config/qt5.nix b/nixos/modules/config/qt5.nix index eabba9ad95f0..24b2a6f9f4a4 100644 --- a/nixos/modules/config/qt5.nix +++ b/nixos/modules/config/qt5.nix @@ -101,4 +101,7 @@ in environment.systemPackages = packages; }; + + # uses relatedPackages + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/i18n/input-method/fcitx.nix b/nixos/modules/i18n/input-method/fcitx.nix index 57960cc365b6..7738581b893a 100644 --- a/nixos/modules/i18n/input-method/fcitx.nix +++ b/nixos/modules/i18n/input-method/fcitx.nix @@ -40,4 +40,7 @@ in }; services.xserver.displayManager.sessionCommands = "${fcitxPackage}/bin/fcitx"; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/i18n/input-method/ibus.nix b/nixos/modules/i18n/input-method/ibus.nix index 92f8c64338a4..c5b0cbc21502 100644 --- a/nixos/modules/i18n/input-method/ibus.nix +++ b/nixos/modules/i18n/input-method/ibus.nix @@ -80,4 +80,7 @@ in ibusPackage ]; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/i18n/input-method/kime.nix b/nixos/modules/i18n/input-method/kime.nix index e462cae2437b..729a665614ae 100644 --- a/nixos/modules/i18n/input-method/kime.nix +++ b/nixos/modules/i18n/input-method/kime.nix @@ -45,5 +45,7 @@ in environment.etc."xdg/kime/config.yaml".text = replaceStrings [ "\\\\" ] [ "\\" ] (builtins.toJSON cfg.config); }; -} + # uses attributes of the linked package + meta.buildDocsInSandbox = false; +} diff --git a/nixos/modules/misc/documentation.nix b/nixos/modules/misc/documentation.nix index 64b1c15086fc..f868e4b709a6 100644 --- a/nixos/modules/misc/documentation.nix +++ b/nixos/modules/misc/documentation.nix @@ -1,19 +1,35 @@ -{ config, lib, pkgs, extendModules, noUserModules, ... }: +{ config, options, lib, pkgs, utils, modules, baseModules, extraModules, modulesPath, ... }: with lib; let cfg = config.documentation; + allOpts = options; /* Modules for which to show options even when not imported. */ extraDocModules = [ ../virtualisation/qemu-vm.nix ]; - /* For the purpose of generating docs, evaluate options with each derivation - in `pkgs` (recursively) replaced by a fake with path "\${pkgs.attribute.path}". - It isn't perfect, but it seems to cover a vast majority of use cases. - Caveat: even if the package is reached by a different means, - the path above will be shown and not e.g. `${config.services.foo.package}`. */ + canCacheDocs = m: + let + f = import m; + instance = f (mapAttrs (n: _: abort "evaluating ${n} for `meta` failed") (functionArgs f)); + in + cfg.nixos.splitOptionDocBuild + && builtins.isPath m + && isFunction f + && instance ? options + && instance.meta.buildDocsInSandbox or true; + + docModules = + let + p = partition canCacheDocs (baseModules ++ extraDocModules); + in + { + lazy = p.right; + eager = p.wrong ++ optionals cfg.nixos.includeAllModules (extraModules ++ modules); + }; + manual = import ../../doc/manual rec { inherit pkgs config; version = config.system.nixos.release; @@ -21,10 +37,17 @@ let extraSources = cfg.nixos.extraModuleSources; options = let - extendNixOS = if cfg.nixos.includeAllModules then extendModules else noUserModules.extendModules; - scrubbedEval = extendNixOS { - modules = extraDocModules; - specialArgs.pkgs = scrubDerivations "pkgs" pkgs; + scrubbedEval = evalModules { + modules = [ { + _module.check = false; + } ] ++ docModules.eager; + specialArgs = { + pkgs = scrubDerivations "pkgs" pkgs; + # allow access to arbitrary options for eager modules, eg for getting + # option types from lazy modules + options = allOpts; + inherit modulesPath utils; + }; }; scrubDerivations = namePrefix: pkgSet: mapAttrs (name: value: @@ -36,6 +59,48 @@ let ) pkgSet; in scrubbedEval.options; + baseOptionsJSON = + let + filter = + builtins.filterSource + (n: t: + (t == "directory" -> baseNameOf n != "tests") + && (t == "file" -> hasSuffix ".nix" n) + ); + in + pkgs.runCommand "lazy-options.json" { + libPath = filter "${toString pkgs.path}/lib"; + pkgsLibPath = filter "${toString pkgs.path}/pkgs/pkgs-lib"; + nixosPath = filter "${toString pkgs.path}/nixos"; + modules = map (p: ''"${removePrefix "${modulesPath}/" (toString p)}"'') docModules.lazy; + } '' + export NIX_STORE_DIR=$TMPDIR/store + export NIX_STATE_DIR=$TMPDIR/state + ${pkgs.nix}/bin/nix-instantiate \ + --show-trace \ + --eval --json --strict \ + --argstr libPath "$libPath" \ + --argstr pkgsLibPath "$pkgsLibPath" \ + --argstr nixosPath "$nixosPath" \ + --arg modules "[ $modules ]" \ + --argstr stateVersion "${options.system.stateVersion.default}" \ + --argstr release "${config.system.nixos.release}" \ + $nixosPath/lib/eval-cacheable-options.nix > $out \ + || { + echo -en "\e[1;31m" + echo 'Cacheable portion of option doc build failed.' + echo 'Usually this means that an option attribute that ends up in documentation (eg' \ + '`default` or `description`) depends on the restricted module arguments' \ + '`config` or `pkgs`.' + echo + echo 'Rebuild your configuration with `--show-trace` to find the offending' \ + 'location. Remove the references to restricted arguments (eg by escaping' \ + 'their antiquotations or adding a `defaultText`) or disable the sandboxed' \ + 'build for the failing module by setting `meta.buildDocsInSandbox = false`.' + echo -en "\e[0m" + exit 1 + } >&2 + ''; }; @@ -191,6 +256,16 @@ in ''; }; + nixos.splitOptionDocBuild = mkOption { + type = types.bool; + default = true; + description = '' + Whether to split the option docs build into a cacheable and an uncacheable part. + Splitting the build can substantially decrease the amount of time needed to build + the manual, but some user modules may be incompatible with this splitting. + ''; + }; + nixos.includeAllModules = mkOption { type = types.bool; default = false; diff --git a/nixos/modules/misc/meta.nix b/nixos/modules/misc/meta.nix index 3dd97cbec235..8e689a63f6bf 100644 --- a/nixos/modules/misc/meta.nix +++ b/nixos/modules/misc/meta.nix @@ -54,6 +54,21 @@ in ''; }; + buildDocsInSandbox = mkOption { + type = types.bool // { + merge = loc: defs: defs; + }; + internal = true; + default = true; + description = '' + Whether to include this module in the split options doc build. + Disable if the module references `config`, `pkgs` or other module + arguments that cannot be evaluated as constants. + + This option should be defined at most once per module. + ''; + }; + }; }; diff --git a/nixos/modules/misc/nixpkgs.nix b/nixos/modules/misc/nixpkgs.nix index 08bc4398555b..2e0c8e4cf2c4 100644 --- a/nixos/modules/misc/nixpkgs.nix +++ b/nixos/modules/misc/nixpkgs.nix @@ -248,4 +248,7 @@ in ) ]; }; + + # needs a full nixpkgs path to import nixpkgs + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/misc/version.nix b/nixos/modules/misc/version.nix index fc0d65d5148e..6c526f6d4f2d 100644 --- a/nixos/modules/misc/version.nix +++ b/nixos/modules/misc/version.nix @@ -119,4 +119,6 @@ in }; + # uses version info nixpkgs, which requires a full nixpkgs path + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/programs/dmrconfig.nix b/nixos/modules/programs/dmrconfig.nix index d2a5117c48ef..73e1b529da9f 100644 --- a/nixos/modules/programs/dmrconfig.nix +++ b/nixos/modules/programs/dmrconfig.nix @@ -7,6 +7,8 @@ let in { meta.maintainers = [ maintainers.etu ]; + # uses relatedPackages + meta.buildDocsInSandbox = false; ###### interface options = { diff --git a/nixos/modules/programs/gnupg.nix b/nixos/modules/programs/gnupg.nix index fe5d7bd834b2..b41f30287ea5 100644 --- a/nixos/modules/programs/gnupg.nix +++ b/nixos/modules/programs/gnupg.nix @@ -149,4 +149,6 @@ in ]; }; + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/programs/tmux.nix b/nixos/modules/programs/tmux.nix index c39908751d29..54c32a463e52 100644 --- a/nixos/modules/programs/tmux.nix +++ b/nixos/modules/programs/tmux.nix @@ -185,4 +185,7 @@ in { imports = [ (lib.mkRenamedOptionModule [ "programs" "tmux" "extraTmuxConf" ] [ "programs" "tmux" "extraConfig" ]) ]; + + # uses relatedPackages + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/backup/sanoid.nix b/nixos/modules/services/backup/sanoid.nix index e70063415ec0..5eb031b2e9f0 100644 --- a/nixos/modules/services/backup/sanoid.nix +++ b/nixos/modules/services/backup/sanoid.nix @@ -51,7 +51,10 @@ let datasetOptions = rec { use_template = mkOption { description = "Names of the templates to use for this dataset."; - type = types.listOf (types.enum (attrNames cfg.templates)); + type = types.listOf (types.str // { + check = (types.enum (attrNames cfg.templates)).check; + description = "configured template name"; + }); default = [ ]; }; useTemplate = use_template; diff --git a/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix b/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix index 4be3e881a9dc..803438b6f7e5 100644 --- a/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix +++ b/nixos/modules/services/desktops/pipewire/pipewire-media-session.nix @@ -29,6 +29,8 @@ in { meta = { maintainers = teams.freedesktop.members; + # uses attributes of the linked package + buildDocsInSandbox = false; }; ###### interface diff --git a/nixos/modules/services/desktops/pipewire/pipewire.nix b/nixos/modules/services/desktops/pipewire/pipewire.nix index 55755ecd6457..372b4785f185 100644 --- a/nixos/modules/services/desktops/pipewire/pipewire.nix +++ b/nixos/modules/services/desktops/pipewire/pipewire.nix @@ -40,6 +40,8 @@ in { meta = { maintainers = teams.freedesktop.members; + # uses attributes of the linked package + buildDocsInSandbox = false; }; ###### interface diff --git a/nixos/modules/services/hardware/thinkfan.nix b/nixos/modules/services/hardware/thinkfan.nix index 4ea829e496e8..1c5b428d5d65 100644 --- a/nixos/modules/services/hardware/thinkfan.nix +++ b/nixos/modules/services/hardware/thinkfan.nix @@ -221,4 +221,7 @@ in { boot.extraModprobeConfig = "options thinkpad_acpi experimental=1 fan_control=1"; }; + + # uses relatedPackages + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/misc/matrix-appservice-irc.nix b/nixos/modules/services/misc/matrix-appservice-irc.nix index 02627e51c932..b041c9c82c56 100644 --- a/nixos/modules/services/misc/matrix-appservice-irc.nix +++ b/nixos/modules/services/misc/matrix-appservice-irc.nix @@ -226,4 +226,7 @@ in { isSystemUser = true; }; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/networking/dnscrypt-proxy2.nix b/nixos/modules/services/networking/dnscrypt-proxy2.nix index dc6a019e9b77..316e6e37f9da 100644 --- a/nixos/modules/services/networking/dnscrypt-proxy2.nix +++ b/nixos/modules/services/networking/dnscrypt-proxy2.nix @@ -118,4 +118,7 @@ in }; }; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/networking/kea.nix b/nixos/modules/services/networking/kea.nix index 4da47f575f79..17b4eb2e283b 100644 --- a/nixos/modules/services/networking/kea.nix +++ b/nixos/modules/services/networking/kea.nix @@ -378,4 +378,6 @@ in ]); meta.maintainers = with maintainers; [ hexa ]; + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/networking/searx.nix b/nixos/modules/services/networking/searx.nix index 9fb06af7442e..6fd81521e7fb 100644 --- a/nixos/modules/services/networking/searx.nix +++ b/nixos/modules/services/networking/searx.nix @@ -228,5 +228,6 @@ in }; meta.maintainers = with maintainers; [ rnhmjoj ]; - + # uses relatedPackages + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/security/vaultwarden/default.nix b/nixos/modules/services/security/vaultwarden/default.nix index 5b951bc85ec0..71088fc4dcd8 100644 --- a/nixos/modules/services/security/vaultwarden/default.nix +++ b/nixos/modules/services/security/vaultwarden/default.nix @@ -179,4 +179,7 @@ in { wantedBy = [ "multi-user.target" ]; }; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/web-apps/dex.nix b/nixos/modules/services/web-apps/dex.nix index f08dd65bdb0f..4d4689a4cf24 100644 --- a/nixos/modules/services/web-apps/dex.nix +++ b/nixos/modules/services/web-apps/dex.nix @@ -112,4 +112,7 @@ in }; }; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/web-apps/gerrit.nix b/nixos/modules/services/web-apps/gerrit.nix index 9ee9dbf1aa49..6bfc67368dd5 100644 --- a/nixos/modules/services/web-apps/gerrit.nix +++ b/nixos/modules/services/web-apps/gerrit.nix @@ -237,4 +237,6 @@ in }; meta.maintainers = with lib.maintainers; [ edef zimbatm ]; + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/web-apps/jirafeau.nix b/nixos/modules/services/web-apps/jirafeau.nix index 83cf224f7d27..a95e2b4f82a9 100644 --- a/nixos/modules/services/web-apps/jirafeau.nix +++ b/nixos/modules/services/web-apps/jirafeau.nix @@ -167,4 +167,7 @@ in "d ${cfg.dataDir}/async/ 0750 ${user} ${group} - -" ]; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index 6692d67081c5..e04b30a7d62d 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -932,4 +932,6 @@ in { ]); meta.doc = ./nextcloud.xml; + # uses relatedPackages + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/web-apps/powerdns-admin.nix b/nixos/modules/services/web-apps/powerdns-admin.nix index ce99b606c318..4661ba80c5d6 100644 --- a/nixos/modules/services/web-apps/powerdns-admin.nix +++ b/nixos/modules/services/web-apps/powerdns-admin.nix @@ -146,4 +146,7 @@ in group = "powerdnsadmin"; }; }; + + # uses attributes of the linked package + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/x11/xserver.nix b/nixos/modules/services/x11/xserver.nix index 24d925734423..f0cabdd4465a 100644 --- a/nixos/modules/services/x11/xserver.nix +++ b/nixos/modules/services/x11/xserver.nix @@ -865,4 +865,6 @@ in }; + # uses relatedPackages + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/system/activation/top-level.nix b/nixos/modules/system/activation/top-level.nix index 501998fa399e..2efe0f05e0c0 100644 --- a/nixos/modules/system/activation/top-level.nix +++ b/nixos/modules/system/activation/top-level.nix @@ -317,4 +317,6 @@ in }; + # uses extendModules to generate a type + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index fa3e25afb03e..29e3aa024dfa 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -999,4 +999,7 @@ in ]; }; + + # uses types of services/x11/xserver.nix + meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/virtualisation/xen-dom0.nix b/nixos/modules/virtualisation/xen-dom0.nix index f8f4af4f6b85..fc640bd947b8 100644 --- a/nixos/modules/virtualisation/xen-dom0.nix +++ b/nixos/modules/virtualisation/xen-dom0.nix @@ -451,4 +451,7 @@ in }; + + # uses relatedPackages + meta.buildDocsInSandbox = false; } From b92a47c87cfc4ff750f69d4de54b016e5f53c449 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 18:57:45 +0100 Subject: [PATCH 08/11] nixos/make-options-doc: add type annotations to mergeJSON.py --- nixos/lib/make-options-doc/mergeJSON.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py index e7f6897c6d0d..b7dfe2b88e7a 100644 --- a/nixos/lib/make-options-doc/mergeJSON.py +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -1,9 +1,12 @@ import collections import json import sys +from typing import Any, Dict, List + +JSON = Dict[str, Any] class Key: - def __init__(self, path): + def __init__(self, path: List[str]): self.path = path def __hash__(self): result = 0 @@ -16,8 +19,8 @@ class Key: Option = collections.namedtuple('Option', ['name', 'value']) # pivot a dict of options keyed by their display name to a dict keyed by their path -def pivot(options): - result = dict() +def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]: + result: Dict[Key, Option] = dict() for (name, opt) in options.items(): result[Key(opt['loc'])] = Option(name, opt) return result @@ -25,8 +28,8 @@ def pivot(options): # pivot back to indexed-by-full-name # like the docbook build we'll just fail if multiple options with differing locs # render to the same option name. -def unpivot(options): - result = dict() +def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: + result: Dict[str, Dict] = dict() for (key, opt) in options.items(): if opt.name in result: raise RuntimeError( From 1301bdb185c4d0d7c30d0400d76eae8669b5b64d Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 19:21:21 +0100 Subject: [PATCH 09/11] nixos/make-options-doc: turn relatedPackages into links link to search.nixos.org instead of pulling package metadata out of pkgs. this lets us cache docs of a few more modules and provides easier access to package info from the HTML manual, but makes the manpage slightly less useful since package description are no longer rendered. --- nixos/lib/make-options-doc/default.nix | 31 +++++++++++-------- nixos/modules/config/qt5.nix | 3 -- nixos/modules/programs/dmrconfig.nix | 2 -- nixos/modules/programs/tmux.nix | 3 -- nixos/modules/services/hardware/thinkfan.nix | 3 -- nixos/modules/services/networking/searx.nix | 2 -- nixos/modules/services/web-apps/nextcloud.nix | 2 -- nixos/modules/virtualisation/xen-dom0.nix | 4 --- 8 files changed, 18 insertions(+), 32 deletions(-) diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 4b40af34b185..cc4ddd55d026 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -55,10 +55,15 @@ let # ../../../lib/options.nix influences. # # Each element of `relatedPackages` can be either - # - a string: that will be interpreted as an attribute name from `pkgs`, - # - a list: that will be interpreted as an attribute path from `pkgs`, - # - an attrset: that can specify `name`, `path`, `package`, `comment` + # - a string: that will be interpreted as an attribute name from `pkgs` and turned into a link + # to search.nixos.org, + # - a list: that will be interpreted as an attribute path from `pkgs` and turned into a link + # to search.nixos.org, + # - an attrset: that can specify `name`, `path`, `comment` # (either of `name`, `path` is required, the rest are optional). + # + # NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists. + # Such checks are not compatible with option docs caching. genRelatedPackages = packages: optName: let unpack = p: if lib.isString p then { name = p; } @@ -68,16 +73,16 @@ let let title = args.title or null; name = args.name or (lib.concatStringsSep "." args.path); - path = args.path or [ args.name ]; - package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}' found while evaluating `relatedPackages' of option `${optName}'") pkgs); - in "" - + "${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})" - + lib.optionalString (!package.meta.available) " [UNAVAILABLE]" - + ": ${package.meta.description or "???"}." - + lib.optionalString (args ? comment) "\n${args.comment}" - # Lots of `longDescription's break DocBook, so we just wrap them into - + lib.optionalString (package.meta ? longDescription) "\n${package.meta.longDescription}" - + ""; + in '' + + + + ${lib.optionalString (title != null) "${title} aka "}pkgs.${name} + + + ${lib.optionalString (args ? comment) "${args.comment}"} + + ''; in "${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}"; # Remove invisible and internal options. diff --git a/nixos/modules/config/qt5.nix b/nixos/modules/config/qt5.nix index 24b2a6f9f4a4..eabba9ad95f0 100644 --- a/nixos/modules/config/qt5.nix +++ b/nixos/modules/config/qt5.nix @@ -101,7 +101,4 @@ in environment.systemPackages = packages; }; - - # uses relatedPackages - meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/programs/dmrconfig.nix b/nixos/modules/programs/dmrconfig.nix index 73e1b529da9f..d2a5117c48ef 100644 --- a/nixos/modules/programs/dmrconfig.nix +++ b/nixos/modules/programs/dmrconfig.nix @@ -7,8 +7,6 @@ let in { meta.maintainers = [ maintainers.etu ]; - # uses relatedPackages - meta.buildDocsInSandbox = false; ###### interface options = { diff --git a/nixos/modules/programs/tmux.nix b/nixos/modules/programs/tmux.nix index 54c32a463e52..c39908751d29 100644 --- a/nixos/modules/programs/tmux.nix +++ b/nixos/modules/programs/tmux.nix @@ -185,7 +185,4 @@ in { imports = [ (lib.mkRenamedOptionModule [ "programs" "tmux" "extraTmuxConf" ] [ "programs" "tmux" "extraConfig" ]) ]; - - # uses relatedPackages - meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/hardware/thinkfan.nix b/nixos/modules/services/hardware/thinkfan.nix index 1c5b428d5d65..4ea829e496e8 100644 --- a/nixos/modules/services/hardware/thinkfan.nix +++ b/nixos/modules/services/hardware/thinkfan.nix @@ -221,7 +221,4 @@ in { boot.extraModprobeConfig = "options thinkpad_acpi experimental=1 fan_control=1"; }; - - # uses relatedPackages - meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/networking/searx.nix b/nixos/modules/services/networking/searx.nix index 6fd81521e7fb..b73f255eb9dd 100644 --- a/nixos/modules/services/networking/searx.nix +++ b/nixos/modules/services/networking/searx.nix @@ -228,6 +228,4 @@ in }; meta.maintainers = with maintainers; [ rnhmjoj ]; - # uses relatedPackages - meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index e04b30a7d62d..6692d67081c5 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -932,6 +932,4 @@ in { ]); meta.doc = ./nextcloud.xml; - # uses relatedPackages - meta.buildDocsInSandbox = false; } diff --git a/nixos/modules/virtualisation/xen-dom0.nix b/nixos/modules/virtualisation/xen-dom0.nix index fc640bd947b8..975eed10cd26 100644 --- a/nixos/modules/virtualisation/xen-dom0.nix +++ b/nixos/modules/virtualisation/xen-dom0.nix @@ -450,8 +450,4 @@ in }; }; - - - # uses relatedPackages - meta.buildDocsInSandbox = false; } From 50954ad1c5847e04fe78fa155fed97ee52dcb9f6 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 19:37:41 +0100 Subject: [PATCH 10/11] nixos/make-options-doc: treat missing descriptions as errors by default this partially solves the problem of "missing description" warnings of the options doc build being lost by nix build, at the cost of failing builds that previously ran. an option to disable this behaviour is provided. --- nixos/doc/manual/default.nix | 13 +++++++++++-- nixos/lib/make-options-doc/default.nix | 4 ++++ nixos/lib/make-options-doc/mergeJSON.py | 20 ++++++++++++++++---- nixos/modules/misc/documentation.nix | 14 ++++++++++++-- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 9bc63686fa3a..52d500c64d33 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -1,4 +1,13 @@ -{ pkgs, options, config, version, revision, extraSources ? [], baseOptionsJSON ? null, prefix ? ../../.. }: +{ pkgs +, options +, config +, version +, revision +, extraSources ? [] +, baseOptionsJSON ? null +, warningsAreErrors ? true +, prefix ? ../../.. +}: with pkgs; @@ -15,7 +24,7 @@ let stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip; optionsDoc = buildPackages.nixosOptionsDoc { - inherit options revision baseOptionsJSON; + inherit options revision baseOptionsJSON warningsAreErrors; transformOptions = opt: opt // { # Clean up declaration sites to not refer to the NixOS source tree. declarations = map stripAnyPrefixes opt.declarations; diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index cc4ddd55d026..57652dd5db1e 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -25,6 +25,9 @@ # used to split the options doc build into a static part (nixos/modules) and a dynamic part # (non-nixos modules imported via configuration.nix, other module sources). , baseOptionsJSON ? null +# instead of printing warnings for eg options with missing descriptions (which may be lost +# by nix build unless -L is given), emit errors instead and fail the build +, warningsAreErrors ? true }: let @@ -121,6 +124,7 @@ in rec { then "cp $options $dst/options.json" else '' ${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \ + ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \ ${baseOptionsJSON} $options \ > $dst/options.json '' diff --git a/nixos/lib/make-options-doc/mergeJSON.py b/nixos/lib/make-options-doc/mergeJSON.py index b7dfe2b88e7a..029787a31586 100644 --- a/nixos/lib/make-options-doc/mergeJSON.py +++ b/nixos/lib/make-options-doc/mergeJSON.py @@ -41,8 +41,10 @@ def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: result[opt.name] = opt.value return result -options = pivot(json.load(open(sys.argv[1], 'r'))) -overrides = pivot(json.load(open(sys.argv[2], 'r'))) +warningsAreErrors = sys.argv[1] == "--warnings-are-errors" +optOffset = 1 if warningsAreErrors else 0 +options = pivot(json.load(open(sys.argv[1 + optOffset], 'r'))) +overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r'))) # fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir for (k, v) in options.items(): @@ -65,10 +67,20 @@ for (k, v) in overrides.items(): cur[ok] = ov # check that every option has a description -# TODO: nixos-rebuild with flakes may hide the warning, maybe turn on -L by default for those? +hasWarnings = False for (k, v) in options.items(): if v.value.get('description', None) is None: - print(f"\x1b[1;31mwarning: option {v.name} has no description\x1b[0m", file=sys.stderr) + severity = "error" if warningsAreErrors else "warning" + hasWarnings = True + print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr) v.value['description'] = "This option has no description." +if hasWarnings and warningsAreErrors: + print( + "\x1b[1;31m" + + "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " + + "to false to ignore these warnings." + + "\x1b[0m", + file=sys.stderr) + sys.exit(1) json.dump(unpivot(options), fp=sys.stdout) diff --git a/nixos/modules/misc/documentation.nix b/nixos/modules/misc/documentation.nix index f868e4b709a6..4451f3026f85 100644 --- a/nixos/modules/misc/documentation.nix +++ b/nixos/modules/misc/documentation.nix @@ -15,7 +15,7 @@ let f = import m; instance = f (mapAttrs (n: _: abort "evaluating ${n} for `meta` failed") (functionArgs f)); in - cfg.nixos.splitOptionDocBuild + cfg.nixos.options.splitBuild && builtins.isPath m && isFunction f && instance ? options @@ -101,6 +101,7 @@ let exit 1 } >&2 ''; + inherit (cfg.nixos.options) warningsAreErrors; }; @@ -256,7 +257,7 @@ in ''; }; - nixos.splitOptionDocBuild = mkOption { + nixos.options.splitBuild = mkOption { type = types.bool; default = true; description = '' @@ -266,6 +267,15 @@ in ''; }; + nixos.options.warningsAreErrors = mkOption { + type = types.bool; + default = true; + description = '' + Treat warning emitted during the option documentation build (eg for missing option + descriptions) as errors. + ''; + }; + nixos.includeAllModules = mkOption { type = types.bool; default = false; From 1511e72b75b49cdeeee68def0c203f997d01bafa Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Dec 2021 20:10:18 +0100 Subject: [PATCH 11/11] nixos/documentation: avoid copying nixpkgs subpaths the docs build should work well even when called from a git checkout of nixpkgs, but should avoid as much work as possible in all cases. if pkgs.path is already a store path we can avoid copying parts of it into the docs build sandbox by wrapping pkgs.path in builtins.storePath --- nixos/modules/misc/documentation.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nixos/modules/misc/documentation.nix b/nixos/modules/misc/documentation.nix index 4451f3026f85..e908a4ae02d7 100644 --- a/nixos/modules/misc/documentation.nix +++ b/nixos/modules/misc/documentation.nix @@ -67,11 +67,15 @@ let (t == "directory" -> baseNameOf n != "tests") && (t == "file" -> hasSuffix ".nix" n) ); + pull = dir: + if isStorePath pkgs.path + then "${builtins.storePath pkgs.path}/${dir}" + else filter "${toString pkgs.path}/${dir}"; in pkgs.runCommand "lazy-options.json" { - libPath = filter "${toString pkgs.path}/lib"; - pkgsLibPath = filter "${toString pkgs.path}/pkgs/pkgs-lib"; - nixosPath = filter "${toString pkgs.path}/nixos"; + libPath = pull "lib"; + pkgsLibPath = pull "pkgs/pkgs-lib"; + nixosPath = pull "nixos"; modules = map (p: ''"${removePrefix "${modulesPath}/" (toString p)}"'') docModules.lazy; } '' export NIX_STORE_DIR=$TMPDIR/store