From 3bb7b3f02e884db944a8a20b4f19227482479b94 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Wed, 3 Oct 2018 18:49:50 +0900 Subject: [PATCH] linux: ability to merge structured configs This should make the composability of kernel configurations more straigthforward. - now distinguish freeform options from tristate ones - will look for a structured config in kernelPatches too one can now access the structuredConfig from a kernel via linux_test.configfile.structuredConfig in order to reinject it into another kernel, no need to rewrite the config from scratch The following merge strategies are used in case of conflict: -- freeform items must be equal or they conflict (mergeEqualOption) -- for tristate (y/m/n) entries, I use the mergeAnswer strategy which takes the best available value, "best" being defined by the user (by default "y" > "m" > "n", e.g. if one entry is both marked "y" and "n", "y" wins) -- if one item is both marked optional/mandatory, mandatory wins (mergeFalseByDefault) --- lib/default.nix | 1 + lib/kernel.nix | 57 +------- nixos/modules/system/boot/kernel_config.nix | 137 ++++++++++++++++++ .../linux/kernel/common-config.nix | 67 ++++----- pkgs/os-specific/linux/kernel/generic.nix | 47 ++++-- 5 files changed, 215 insertions(+), 94 deletions(-) create mode 100644 nixos/modules/system/boot/kernel_config.nix diff --git a/lib/default.nix b/lib/default.nix index d400907ebb0c..5ae3667406dd 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -32,6 +32,7 @@ let modules = callLibs ./modules.nix; options = callLibs ./options.nix; types = callLibs ./types.nix; + kernel = callLibs ./kernel.nix; # constants licenses = callLibs ./licenses.nix; diff --git a/lib/kernel.nix b/lib/kernel.nix index 45b33aea7b87..14783ae97393 100644 --- a/lib/kernel.nix +++ b/lib/kernel.nix @@ -1,57 +1,16 @@ -{ lib -# we pass the kernel version here to keep a nice syntax `whenOlder "4.13"` -# kernelVersion, e.g., config.boot.kernelPackages.version -, version -, mkValuePreprocess ? null -}: +{ lib }: with lib; rec { - # Common patterns - when = cond: opt: if cond then opt else null; - whenAtLeast = ver: when (versionAtLeast version ver); - whenOlder = ver: when (versionOlder version ver); - whenBetween = verLow: verHigh: when (versionAtLeast version verLow && versionOlder version verHigh); + # Keeping these around in case we decide to change this horrible implementation :) - option = x: if x == null then null else "?${x}"; - yes = "y"; - no = "n"; - module = "m"; + option = x: + x // { optional = true; }; - mkValue = val: - let - isNumber = c: elem c ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"]; - in - if val == "" then "\"\"" - else if val == yes || val == module || val == no then val - else if all isNumber (stringToCharacters val) then val - else if substring 0 2 val == "0x" then val - else val; # FIXME: fix quoting one day + yes = { tristate = "y"; }; + no = { tristate = "n"; }; + module = { tristate = "m"; }; + freeform = x: { freeform = x; }; - - # generate nix intermediate kernel config file of the form - # - # VIRTIO_MMIO m - # VIRTIO_BLK y - # VIRTIO_CONSOLE n - # NET_9P_VIRTIO? y - # - # Use mkValuePreprocess to preprocess option values, aka mark 'modules' as - # 'yes' or vice-versa - # Borrowed from copumpkin https://github.com/NixOS/nixpkgs/pull/12158 - # returns a string, expr should be an attribute set - generateNixKConf = exprs: mkValuePreprocess: - let - mkConfigLine = key: rawval: - let - val = if builtins.isFunction mkValuePreprocess then mkValuePreprocess rawval else rawval; - in - if val == null - then "" - else if hasPrefix "?" val - then "${key}? ${mkValue (removePrefix "?" val)}\n" - else "${key} ${mkValue val}\n"; - mkConf = cfg: concatStrings (mapAttrsToList mkConfigLine cfg); - in mkConf exprs; } diff --git a/nixos/modules/system/boot/kernel_config.nix b/nixos/modules/system/boot/kernel_config.nix new file mode 100644 index 000000000000..fbbd0982b2c6 --- /dev/null +++ b/nixos/modules/system/boot/kernel_config.nix @@ -0,0 +1,137 @@ +{ lib, config, ... }: + +with lib; +let + findWinner = candidates: winner: + any (x: x == winner) candidates; + + # winners is an ordered list where first item wins over 2nd etc + mergeAnswer = winners: locs: defs: + let + values = map (x: x.value) defs; + freeformAnswer = intersectLists values winners; + inter = intersectLists values winners; + winner = head winners; + in + if defs == [] then abort "This case should never happen." + else if winner == [] then abort "Give a valid list of winner" + else if inter == [] then mergeOneOption locs defs + else if findWinner values winner then + winner + else + mergeAnswer (tail winners) locs defs; + + mergeFalseByDefault = locs: defs: + if defs == [] then abort "This case should never happen." + else if any (x: x == false) defs then false + else true; + + kernelItem = types.submodule { + options = { + tristate = mkOption { + type = types.enum [ "y" "m" "n" null ] // { + merge = mergeAnswer [ "y" "m" "n" ]; + }; + default = null; + internal = true; + visible = true; + description = '' + Use this field for tristate kernel options expecting a "y" or "m" or "n". + ''; + }; + + freeform = mkOption { + type = types.nullOr types.str // { + merge = mergeEqualOption; + }; + default = null; + example = ''MMC_BLOCK_MINORS.freeform = "32";''; + description = '' + Freeform description of a kernel configuration item value. + ''; + }; + + optional = mkOption { + type = types.bool // { merge = mergeFalseByDefault; }; + default = false; + description = '' + Wether option should generate a failure when unused. + ''; + }; + }; + }; + + mkValue = with lib; val: + let + isNumber = c: elem c ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"]; + + in + if (val == "") then "\"\"" + else if val == "y" || val == "m" || val == "n" then val + else if all isNumber (stringToCharacters val) then val + else if substring 0 2 val == "0x" then val + else val; # FIXME: fix quoting one day + + + # generate nix intermediate kernel config file of the form + # + # VIRTIO_MMIO m + # VIRTIO_BLK y + # VIRTIO_CONSOLE n + # NET_9P_VIRTIO? y + # + # Borrowed from copumpkin https://github.com/NixOS/nixpkgs/pull/12158 + # returns a string, expr should be an attribute set + # Use mkValuePreprocess to preprocess option values, aka mark 'modules' as 'yes' or vice-versa + # use the identity if you don't want to override the configured values + generateNixKConf = exprs: + let + mkConfigLine = key: item: + let + val = if item.freeform != null then item.freeform else item.tristate; + in + if val == null + then "" + else if (item.optional) + then "${key}? ${mkValue val}\n" + else "${key} ${mkValue val}\n"; + + mkConf = cfg: concatStrings (mapAttrsToList mkConfigLine cfg); + in mkConf exprs; + +in +{ + + options = { + + intermediateNixConfig = mkOption { + readOnly = true; + type = types.lines; + example = '' + USB? y + DEBUG n + ''; + description = '' + The result of converting the structured kernel configuration in settings + to an intermediate string that can be parsed by generate-config.pl to + answer the kernel `make defconfig`. + ''; + }; + + settings = mkOption { + type = types.attrsOf kernelItem; + example = literalExample '' with lib.kernel; { + "9P_NET" = yes; + USB = optional yes; + MMC_BLOCK_MINORS = freeform "32"; + }''; + description = '' + Structured kernel configuration. + ''; + }; + }; + + config = { + intermediateNixConfig = generateNixKConf config.settings; + }; +} diff --git a/pkgs/os-specific/linux/kernel/common-config.nix b/pkgs/os-specific/linux/kernel/common-config.nix index ddd1e9828d51..bdcad8c2383c 100644 --- a/pkgs/os-specific/linux/kernel/common-config.nix +++ b/pkgs/os-specific/linux/kernel/common-config.nix @@ -12,25 +12,19 @@ # Configuration { stdenv, version -# to let user override values, aka converting modules to included and vice-versa -, mkValueOverride ? null - -# new extraConfig as a flattened set -, structuredExtraConfig ? {} - -# legacy extraConfig as string -, extraConfig ? "" - , features ? { grsecurity = false; xen_dom0 = false; } }: -assert (mkValueOverride == null) || (builtins.isFunction mkValueOverride); - with stdenv.lib; -with import ../../../../lib/kernel.nix { inherit (stdenv) lib; inherit version; }; + with import ../../../../lib/kernel.nix { inherit (stdenv) lib; }; let + # Common patterns/legacy + when = cond: opt: if cond then opt else null; + whenAtLeast = ver: mkIf (versionAtLeast version ver); + whenOlder = ver: mkIf (versionOlder version ver); + whenBetween = verLow: verHigh: mkIf (versionAtLeast version verLow && versionOlder version verHigh); # configuration items have to be part of a subattrs flattenKConf = nested: mapAttrs (_: head) (zipAttrs (attrValues nested)); @@ -46,7 +40,7 @@ let DEBUG_NX_TEST = whenOlder "4.11" no; CPU_NOTIFIER_ERROR_INJECT = whenOlder "4.4" (option no); DEBUG_STACK_USAGE = no; - DEBUG_STACKOVERFLOW = when (!features.grsecurity) no; + DEBUG_STACKOVERFLOW = mkIf (!features.grsecurity) no; RCU_TORTURE_TEST = no; SCHEDSTATS = no; DETECT_HUNG_TASK = yes; @@ -114,7 +108,7 @@ let IP_DCCP_CCID3 = no; # experimental CLS_U32_PERF = yes; CLS_U32_MARK = yes; - BPF_JIT = when (stdenv.hostPlatform.system == "x86_64-linux") yes; + BPF_JIT = mkIf (stdenv.hostPlatform.system == "x86_64-linux") yes; WAN = yes; # Required by systemd per-cgroup firewalling CGROUP_BPF = option yes; @@ -184,7 +178,7 @@ let FB_VESA = yes; FRAMEBUFFER_CONSOLE = yes; FRAMEBUFFER_CONSOLE_ROTATION = yes; - FB_GEODE = when (stdenv.hostPlatform.system == "i686-linux") yes; + FB_GEODE = mkIf (stdenv.hostPlatform.system == "i686-linux") yes; }; video = { @@ -239,7 +233,7 @@ let }; usb = { - USB_DEBUG = option (whenOlder "4.18" no); + USB_DEBUG = { optional = true; tristate = whenOlder "4.18" "n";}; USB_EHCI_ROOT_HUB_TT = yes; # Root Hub Transaction Translators USB_EHCI_TT_NEWSCHED = yes; # Improved transaction translator scheduling }; @@ -250,7 +244,7 @@ let FANOTIFY = yes; TMPFS = yes; TMPFS_POSIX_ACL = yes; - FS_ENCRYPTION = option (whenAtLeast "4.9" module); + FS_ENCRYPTION = { optional = true; tristate = whenAtLeast "4.9" "m"; }; EXT2_FS_XATTR = yes; EXT2_FS_POSIX_ACL = yes; @@ -262,7 +256,7 @@ let EXT4_FS_POSIX_ACL = yes; EXT4_FS_SECURITY = yes; - EXT4_ENCRYPTION = option ((if (versionOlder version "4.8") then module else yes)); + EXT4_ENCRYPTION = { optional = true; tristate = if (versionOlder version "4.8") then "m" else "y"; }; REISERFS_FS_XATTR = option yes; REISERFS_FS_POSIX_ACL = option yes; @@ -324,7 +318,7 @@ let # Native Language Support modules, needed by some filesystems NLS = yes; - NLS_DEFAULT = "utf8"; + NLS_DEFAULT = freeform "utf8"; NLS_UTF8 = module; NLS_CODEPAGE_437 = module; # VFAT default for the codepage= mount option NLS_ISO8859_1 = module; # VFAT default for the iocharset= mount option @@ -334,13 +328,13 @@ let security = { # Detect writes to read-only module pages - DEBUG_SET_MODULE_RONX = option (whenOlder "4.11" yes); + DEBUG_SET_MODULE_RONX = { optional = true; tristate = whenOlder "4.11" "y"; }; RANDOMIZE_BASE = option yes; STRICT_DEVMEM = option yes; # Filter access to /dev/mem - SECURITY_SELINUX_BOOTPARAM_VALUE = "0"; # Disable SELinux by default + SECURITY_SELINUX_BOOTPARAM_VALUE = freeform "0"; # Disable SELinux by default # Prevent processes from ptracing non-children processes SECURITY_YAMA = option yes; - DEVKMEM = when (!features.grsecurity) no; # Disable /dev/kmem + DEVKMEM = mkIf (!features.grsecurity) no; # Disable /dev/kmem USER_NS = yes; # Support for user namespaces @@ -350,7 +344,7 @@ let } // optionalAttrs (!stdenv.hostPlatform.isAarch32) { # Detect buffer overflows on the stack - CC_STACKPROTECTOR_REGULAR = option (whenOlder "4.18" yes); + CC_STACKPROTECTOR_REGULAR = {optional = true; tristate = whenOlder "4.18" "y";}; }; microcode = { @@ -407,8 +401,8 @@ let FTRACE_SYSCALLS = yes; SCHED_TRACER = yes; STACK_TRACER = yes; - UPROBE_EVENT = option (whenOlder "4.11" yes); - UPROBE_EVENTS = option (whenAtLeast "4.11" yes); + UPROBE_EVENT = { optional = true; tristate = whenOlder "4.11" "y";}; + UPROBE_EVENTS = { optional = true; tristate = whenAtLeast "4.11" "y";}; BPF_SYSCALL = whenAtLeast "4.4" yes; BPF_EVENTS = whenAtLeast "4.4" yes; FUNCTION_PROFILER = yes; @@ -418,13 +412,13 @@ let virtualisation = { PARAVIRT = option yes; - HYPERVISOR_GUEST = when (!features.grsecurity) yes; + HYPERVISOR_GUEST = mkIf (!features.grsecurity) yes; PARAVIRT_SPINLOCKS = option yes; KVM_APIC_ARCHITECTURE = whenOlder "4.8" yes; KVM_ASYNC_PF = yes; - KVM_COMPAT = option (whenBetween "4.0" "4.12" yes); - KVM_DEVICE_ASSIGNMENT = option (whenBetween "3.10" "4.12" yes); + KVM_COMPAT = { optional = true; tristate = whenBetween "4.0" "4.12" "y"; }; + KVM_DEVICE_ASSIGNMENT = { optional = true; tristate = whenBetween "3.10" "4.12" "y"; }; KVM_GENERIC_DIRTYLOG_READ_PROTECT = whenAtLeast "4.0" yes; KVM_GUEST = when (!features.grsecurity) yes; KVM_MMIO = yes; @@ -432,9 +426,9 @@ let KSM = yes; VIRT_DRIVERS = yes; # We nneed 64 GB (PAE) support for Xen guest support - HIGHMEM64G = option (when (!stdenv.is64bit) yes); + HIGHMEM64G = { optional = true; tristate = mkIf (!stdenv.is64bit) "y";}; - VFIO_PCI_VGA = when stdenv.is64bit yes; + VFIO_PCI_VGA = mkIf stdenv.is64bit yes; } // optionalAttrs (stdenv.isx86_64 || stdenv.isi686) ({ XEN = option yes; @@ -542,8 +536,8 @@ let CRYPTO_TEST = option no; EFI_TEST = option no; GLOB_SELFTEST = option no; - DRM_DEBUG_MM_SELFTEST = option (whenOlder "4.18" no); - LNET_SELFTEST = option (whenOlder "4.18" no); + DRM_DEBUG_MM_SELFTEST = { optional = true; tristate = whenOlder "4.18" "n";}; + LNET_SELFTEST = { optional = true; tristate = whenOlder "4.18" "n";}; LOCK_TORTURE_TEST = option no; MTD_TESTS = option no; NOTIFIER_ERROR_INJECTION = option no; @@ -598,7 +592,7 @@ let AIC79XX_DEBUG_ENABLE = no; AIC7XXX_DEBUG_ENABLE = no; AIC94XX_DEBUG = no; - B43_PCMCIA = option (whenOlder "4.4" yes); + B43_PCMCIA = { optional=true; tristate = whenOlder "4.4" "y";}; BLK_DEV_INTEGRITY = yes; @@ -651,7 +645,7 @@ let # GPIO on Intel Bay Trail, for some Chromebook internal eMMC disks PINCTRL_BAYTRAIL = yes; # 8 is default. Modern gpt tables on eMMC may go far beyond 8. - MMC_BLOCK_MINORS = "32"; + MMC_BLOCK_MINORS = freeform "32"; REGULATOR = yes; # Voltage and Current Regulator Support RC_DEVICES = option yes; # Enable IR devices @@ -698,7 +692,8 @@ let # Bump the maximum number of CPUs to support systems like EC2 x1.* # instances and Xeon Phi. - NR_CPUS = "384"; + NR_CPUS = freeform "384"; }; }; -in (generateNixKConf ((flattenKConf options) // structuredExtraConfig) mkValueOverride) + extraConfig +in + flattenKConf options diff --git a/pkgs/os-specific/linux/kernel/generic.nix b/pkgs/os-specific/linux/kernel/generic.nix index 30878d1b96c6..a41f1eb989bb 100644 --- a/pkgs/os-specific/linux/kernel/generic.nix +++ b/pkgs/os-specific/linux/kernel/generic.nix @@ -47,7 +47,6 @@ , preferBuiltin ? stdenv.hostPlatform.platform.kernelPreferBuiltin or false , kernelArch ? stdenv.hostPlatform.platform.kernelArch -, mkValueOverride ? null , ... }: @@ -68,20 +67,26 @@ let ia32Emulation = true; } // features) kernelPatches; - intermediateNixConfig = import ./common-config.nix { - inherit stdenv version structuredExtraConfig mkValueOverride; - - # append extraConfig for backwards compatibility but also means the user can't override the kernelExtraConfig part - extraConfig = extraConfig + lib.optionalString (stdenv.hostPlatform.platform ? kernelExtraConfig) stdenv.hostPlatform.platform.kernelExtraConfig; + commonStructuredConfig = import ./common-config.nix { + inherit stdenv version ; features = kernelFeatures; # Ensure we know of all extra patches, etc. }; - kernelConfigFun = baseConfig: + # extra config in legacy string format + extraConfig = extraConfig + lib.optionalString (stdenv.hostPlatform.platform ? kernelExtraConfig) stdenv.hostPlatform.platform.kernelExtraConfig; + + intermediateNixConfig = configfile.moduleStructuredConfig.intermediateNixConfig; + + structuredConfigFromPatches = + map ({extraStructuredConfig ? {}, ...}: {settings=extraStructuredConfig;}) kernelPatches; + + # appends kernel patches extraConfig + kernelConfigFun = baseConfigStr: let configFromPatches = map ({extraConfig ? "", ...}: extraConfig) kernelPatches; - in lib.concatStringsSep "\n" ([baseConfig] ++ configFromPatches); + in lib.concatStringsSep "\n" ([baseConfigStr] ++ configFromPatches); configfile = stdenv.mkDerivation { inherit ignoreConfigErrors autoModules preferBuiltin kernelArch; @@ -131,7 +136,30 @@ let installPhase = "mv $buildRoot/.config $out"; enableParallelBuilding = true; - }; + + passthru = rec { + + module = import ../../../../nixos/modules/system/boot/kernel_config.nix; + # used also in apache + # { modules = [ { options = res.options; config = svc.config or svc; } ]; + # check = false; + # The result is a set of two attributes + moduleStructuredConfig = (lib.evalModules { + modules = [ + module + { settings = commonStructuredConfig; } + { settings = structuredExtraConfig; } + ] + ++ structuredConfigFromPatches + ; + }).config; + + # + structuredConfig = moduleStructuredConfig.settings; + }; + + + }; # end of configfile derivation kernel = (callPackage ./manual-config.nix {}) { inherit version modDirVersion src kernelPatches stdenv extraMeta configfile; @@ -141,6 +169,7 @@ let passthru = { features = kernelFeatures; + inherit commonStructuredConfig; passthru = kernel.passthru // (removeAttrs passthru [ "passthru" ]); };