linux: ability to merge structured configs

This should make the composability of kernel configurations more straigthforward.

- now distinguish freeform options from tristate ones
- will look for a structured config in kernelPatches too
one can now access the structuredConfig from a kernel via linux_test.configfile.structuredConfig
in order to reinject it into another kernel, no need to rewrite the config from scratch

The following merge strategies are used in case of conflict:
-- freeform items must be equal or they conflict (mergeEqualOption)
-- for tristate (y/m/n) entries, I use the mergeAnswer strategy which takes the best available value, "best" being defined by the user (by default "y" > "m" > "n", e.g. if one entry is both marked "y" and "n", "y" wins)
-- if one item is both marked optional/mandatory, mandatory wins (mergeFalseByDefault)

lib/default.nix

@@ -32,6 +32,7 @@ let
     modules = callLibs ./modules.nix;
     options = callLibs ./options.nix;
     types = callLibs ./types.nix;
+    kernel = callLibs ./kernel.nix;
 
     # constants
     licenses = callLibs ./licenses.nix;

lib/kernel.nix

@@ -1,57 +1,16 @@
-{ lib
+{ lib }:
-# we pass the kernel version here to keep a nice syntax `whenOlder "4.13"`
-# kernelVersion, e.g., config.boot.kernelPackages.version
-, version
-, mkValuePreprocess ? null
-}:
 
 with lib;
 rec {
-  # Common patterns
+
-  when        = cond: opt: if cond then opt else null;
-  whenAtLeast = ver: when (versionAtLeast version ver);
-  whenOlder   = ver: when (versionOlder version ver);
-  whenBetween = verLow: verHigh: when (versionAtLeast version verLow && versionOlder version verHigh);
 
   # Keeping these around in case we decide to change this horrible implementation :)
-  option = x: if x == null then null else "?${x}";
+  option = x:
-  yes    = "y";
+      x // { optional = true; };
-  no     = "n";
-  module = "m";
 
-  mkValue = val:
+  yes      = { tristate    = "y"; };
-  let
+  no       = { tristate    = "n"; };
-    isNumber = c: elem c ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"];
+  module   = { tristate    = "m"; };
-  in
+  freeform = x: { freeform = x; };
-    if val == "" then "\"\""
-    else if val == yes || val == module || val == no then val
-    else if all isNumber (stringToCharacters val) then val
-    else if substring 0 2 val == "0x" then val
-    else val; # FIXME: fix quoting one day
 
-
-  # generate nix intermediate kernel config file of the form
-  #
-  #       VIRTIO_MMIO m
-  #       VIRTIO_BLK y
-  #       VIRTIO_CONSOLE n
-  #       NET_9P_VIRTIO? y
-  #
-  # Use mkValuePreprocess to preprocess option values, aka mark 'modules' as
-  # 'yes' or vice-versa
-  # Borrowed from copumpkin https://github.com/NixOS/nixpkgs/pull/12158
-  # returns a string, expr should be an attribute set
-  generateNixKConf = exprs: mkValuePreprocess:
-  let
-    mkConfigLine = key: rawval:
-    let
-      val = if builtins.isFunction mkValuePreprocess then mkValuePreprocess rawval else rawval;
-    in
-      if val == null
-        then ""
-        else if hasPrefix "?" val
-          then "${key}? ${mkValue (removePrefix "?" val)}\n"
-          else "${key} ${mkValue val}\n";
-    mkConf = cfg: concatStrings (mapAttrsToList mkConfigLine cfg);
-  in mkConf exprs;
 }

nixos/modules/system/boot/kernel_config.nix

@@ -0,0 +1,137 @@
+{ lib, config, ... }:
+
+with lib;
+let
+  findWinner = candidates: winner:
+    any (x: x == winner) candidates;
+
+  # winners is an ordered list where first item wins over 2nd etc
+  mergeAnswer = winners: locs: defs:
+    let
+      values = map (x: x.value) defs;
+      freeformAnswer = intersectLists values winners;
+      inter = intersectLists values winners;
+      winner = head winners;
+    in
+    if defs == [] then abort "This case should never happen."
+    else if winner == [] then abort "Give a valid list of winner"
+    else if inter == [] then mergeOneOption locs defs
+    else if findWinner values winner then
+      winner
+    else
+      mergeAnswer (tail winners) locs defs;
+
+  mergeFalseByDefault = locs: defs:
+    if defs == [] then abort "This case should never happen."
+    else if any (x: x == false) defs then false
+    else true;
+
+  kernelItem = types.submodule {
+    options = {
+      tristate = mkOption {
+        type = types.enum [ "y" "m" "n" null ] // {
+          merge = mergeAnswer [ "y" "m" "n" ];
+        };
+        default = null;
+        internal = true;
+        visible = true;
+        description = ''
+          Use this field for tristate kernel options expecting a "y" or "m" or "n".
+        '';
+      };
+
+      freeform = mkOption {
+        type = types.nullOr types.str // {
+          merge = mergeEqualOption;
+        };
+        default = null;
+        example = ''MMC_BLOCK_MINORS.freeform = "32";'';
+        description = ''
+          Freeform description of a kernel configuration item value.
+        '';
+      };
+
+      optional = mkOption {
+        type = types.bool // { merge = mergeFalseByDefault; };
+        default = false;
+        description = ''
+          Wether option should generate a failure when unused.
+        '';
+      };
+    };
+  };
+
+  mkValue = with lib; val:
+  let
+    isNumber = c: elem c ["0" "1" "2" "3" "4" "5" "6" "7" "8" "9"];
+
+  in
+    if (val == "") then "\"\""
+    else if val == "y" || val == "m" || val == "n" then val
+    else if all isNumber (stringToCharacters val) then val
+    else if substring 0 2 val == "0x" then val
+    else val; # FIXME: fix quoting one day
+
+
+  # generate nix intermediate kernel config file of the form
+  #
+  #       VIRTIO_MMIO m
+  #       VIRTIO_BLK y
+  #       VIRTIO_CONSOLE n
+  #       NET_9P_VIRTIO? y
+  #
+  # Borrowed from copumpkin https://github.com/NixOS/nixpkgs/pull/12158
+  # returns a string, expr should be an attribute set
+  # Use mkValuePreprocess to preprocess option values, aka mark 'modules' as 'yes' or vice-versa
+  # use the identity if you don't want to override the configured values
+  generateNixKConf = exprs:
+  let
+    mkConfigLine = key: item:
+      let
+        val = if item.freeform != null then item.freeform else item.tristate;
+      in
+        if val == null
+          then ""
+          else if (item.optional)
+            then "${key}? ${mkValue val}\n"
+            else "${key} ${mkValue val}\n";
+
+    mkConf = cfg: concatStrings (mapAttrsToList mkConfigLine cfg);
+  in mkConf exprs;
+
+in
+{
+
+  options = {
+
+    intermediateNixConfig = mkOption {
+      readOnly = true;
+      type = types.lines;
+      example = ''
+        USB? y
+        DEBUG n
+      '';
+      description = ''
+        The result of converting the structured kernel configuration in settings
+        to an intermediate string that can be parsed by generate-config.pl to
+        answer the kernel `make defconfig`.
+      '';
+    };
+
+    settings = mkOption {
+      type = types.attrsOf kernelItem;
+      example = literalExample '' with lib.kernel; {
+        "9P_NET" = yes;
+        USB = optional yes;
+        MMC_BLOCK_MINORS = freeform "32";
+      }'';
+      description = ''
+        Structured kernel configuration.
+      '';
+    };
+  };
+
+  config = {
+    intermediateNixConfig = generateNixKConf config.settings;
+  };
+}

pkgs/os-specific/linux/kernel/common-config.nix

@@ -12,25 +12,19 @@
 # Configuration
 { stdenv, version
 
-# to let user override values, aka converting modules to included and vice-versa
-, mkValueOverride ? null
-
-# new extraConfig as a flattened set
-, structuredExtraConfig ? {}
-
-# legacy extraConfig as string
-, extraConfig ? ""
-
 , features ? { grsecurity = false; xen_dom0 = false; }
 }:
 
-assert (mkValueOverride == null) || (builtins.isFunction mkValueOverride);
-
 with stdenv.lib;
 
-with import ../../../../lib/kernel.nix { inherit (stdenv) lib; inherit version; };
+  with import ../../../../lib/kernel.nix { inherit (stdenv) lib; };
 
 let
+  # Common patterns/legacy
+  when = cond: opt: if cond then opt else null;
+  whenAtLeast = ver: mkIf (versionAtLeast version ver);
+  whenOlder   = ver: mkIf (versionOlder version ver);
+  whenBetween = verLow: verHigh: mkIf (versionAtLeast version verLow && versionOlder version verHigh);
 
   # configuration items have to be part of a subattrs
   flattenKConf =  nested: mapAttrs (_: head) (zipAttrs (attrValues nested));
@@ -46,7 +40,7 @@ let
       DEBUG_NX_TEST             = whenOlder "4.11" no;
       CPU_NOTIFIER_ERROR_INJECT = whenOlder "4.4" (option no);
       DEBUG_STACK_USAGE         = no;
-      DEBUG_STACKOVERFLOW       = when (!features.grsecurity) no;
+      DEBUG_STACKOVERFLOW       = mkIf (!features.grsecurity) no;
       RCU_TORTURE_TEST          = no;
       SCHEDSTATS                = no;
       DETECT_HUNG_TASK          = yes;
@@ -114,7 +108,7 @@ let
       IP_DCCP_CCID3      = no; # experimental
       CLS_U32_PERF       = yes;
       CLS_U32_MARK       = yes;
-      BPF_JIT            = when (stdenv.hostPlatform.system == "x86_64-linux") yes;
+      BPF_JIT            = mkIf (stdenv.hostPlatform.system == "x86_64-linux") yes;
       WAN                = yes;
       # Required by systemd per-cgroup firewalling
       CGROUP_BPF                  = option yes;
@@ -184,7 +178,7 @@ let
       FB_VESA             = yes;
       FRAMEBUFFER_CONSOLE = yes;
       FRAMEBUFFER_CONSOLE_ROTATION = yes;
-      FB_GEODE            = when (stdenv.hostPlatform.system == "i686-linux") yes;
+      FB_GEODE            = mkIf (stdenv.hostPlatform.system == "i686-linux") yes;
     };
 
     video = {
@@ -239,7 +233,7 @@ let
     };
 
     usb = {
-      USB_DEBUG            = option (whenOlder "4.18" no);
+      USB_DEBUG = { optional = true; tristate = whenOlder "4.18" "n";};
       USB_EHCI_ROOT_HUB_TT = yes; # Root Hub Transaction Translators
       USB_EHCI_TT_NEWSCHED = yes; # Improved transaction translator scheduling
     };
@@ -250,7 +244,7 @@ let
       FANOTIFY        = yes;
       TMPFS           = yes;
       TMPFS_POSIX_ACL = yes;
-      FS_ENCRYPTION   = option (whenAtLeast "4.9" module);
+      FS_ENCRYPTION   = { optional = true; tristate = whenAtLeast "4.9" "m"; };
 
       EXT2_FS_XATTR     = yes;
       EXT2_FS_POSIX_ACL = yes;
@@ -262,7 +256,7 @@ let
 
       EXT4_FS_POSIX_ACL = yes;
       EXT4_FS_SECURITY  = yes;
-      EXT4_ENCRYPTION   = option ((if (versionOlder version "4.8") then module else yes));
+      EXT4_ENCRYPTION   = { optional = true; tristate = if (versionOlder version "4.8") then "m" else "y"; };
 
       REISERFS_FS_XATTR     = option yes;
       REISERFS_FS_POSIX_ACL = option yes;
@@ -324,7 +318,7 @@ let
 
       # Native Language Support modules, needed by some filesystems
       NLS              = yes;
-      NLS_DEFAULT      = "utf8";
+      NLS_DEFAULT      = freeform "utf8";
       NLS_UTF8         = module;
       NLS_CODEPAGE_437 = module; # VFAT default for the codepage= mount option
       NLS_ISO8859_1    = module; # VFAT default for the iocharset= mount option
@@ -334,13 +328,13 @@ let
 
     security = {
       # Detect writes to read-only module pages
-      DEBUG_SET_MODULE_RONX            = option (whenOlder "4.11" yes);
+      DEBUG_SET_MODULE_RONX            = { optional = true; tristate = whenOlder "4.11" "y"; };
       RANDOMIZE_BASE                   = option yes;
       STRICT_DEVMEM                    = option yes; # Filter access to /dev/mem
-      SECURITY_SELINUX_BOOTPARAM_VALUE = "0"; # Disable SELinux by default
+      SECURITY_SELINUX_BOOTPARAM_VALUE = freeform "0"; # Disable SELinux by default
       # Prevent processes from ptracing non-children processes
       SECURITY_YAMA                    = option yes;
-      DEVKMEM                          = when (!features.grsecurity) no; # Disable /dev/kmem
+      DEVKMEM                          = mkIf (!features.grsecurity) no; # Disable /dev/kmem
 
       USER_NS                          = yes; # Support for user namespaces
 
@@ -350,7 +344,7 @@ let
     } // optionalAttrs (!stdenv.hostPlatform.isAarch32) {
 
       # Detect buffer overflows on the stack
-      CC_STACKPROTECTOR_REGULAR = option (whenOlder "4.18" yes);
+      CC_STACKPROTECTOR_REGULAR = {optional = true; tristate = whenOlder "4.18" "y";};
     };
 
     microcode = {
@@ -407,8 +401,8 @@ let
       FTRACE_SYSCALLS       = yes;
       SCHED_TRACER          = yes;
       STACK_TRACER          = yes;
-      UPROBE_EVENT          = option (whenOlder "4.11" yes);
+      UPROBE_EVENT          = { optional = true; tristate = whenOlder "4.11" "y";};
-      UPROBE_EVENTS         = option (whenAtLeast "4.11" yes);
+      UPROBE_EVENTS         = { optional = true; tristate = whenAtLeast "4.11" "y";};
       BPF_SYSCALL           = whenAtLeast "4.4" yes;
       BPF_EVENTS            = whenAtLeast "4.4" yes;
       FUNCTION_PROFILER     = yes;
@@ -418,13 +412,13 @@ let
     virtualisation = {
       PARAVIRT = option yes;
 
-      HYPERVISOR_GUEST = when (!features.grsecurity) yes;
+      HYPERVISOR_GUEST = mkIf (!features.grsecurity) yes;
       PARAVIRT_SPINLOCKS  = option yes;
 
       KVM_APIC_ARCHITECTURE             = whenOlder "4.8" yes;
       KVM_ASYNC_PF                      = yes;
-      KVM_COMPAT                        = option (whenBetween "4.0" "4.12"  yes);
+      KVM_COMPAT = { optional = true; tristate = whenBetween "4.0" "4.12" "y"; };
-      KVM_DEVICE_ASSIGNMENT             = option (whenBetween "3.10" "4.12" yes);
+      KVM_DEVICE_ASSIGNMENT  = { optional = true; tristate = whenBetween "3.10" "4.12" "y"; };
       KVM_GENERIC_DIRTYLOG_READ_PROTECT = whenAtLeast "4.0"  yes;
       KVM_GUEST                         = when (!features.grsecurity) yes;
       KVM_MMIO                          = yes;
@@ -432,9 +426,9 @@ let
       KSM = yes;
       VIRT_DRIVERS = yes;
       # We nneed 64 GB (PAE) support for Xen guest support
-      HIGHMEM64G = option (when (!stdenv.is64bit) yes);
+      HIGHMEM64G = { optional = true; tristate = mkIf (!stdenv.is64bit) "y";};
 
-      VFIO_PCI_VGA = when stdenv.is64bit yes;
+      VFIO_PCI_VGA = mkIf stdenv.is64bit yes;
 
     } // optionalAttrs (stdenv.isx86_64 || stdenv.isi686) ({
       XEN = option yes;
@@ -542,8 +536,8 @@ let
       CRYPTO_TEST              = option no;
       EFI_TEST                 = option no;
       GLOB_SELFTEST            = option no;
-      DRM_DEBUG_MM_SELFTEST    = option (whenOlder "4.18" no);
+      DRM_DEBUG_MM_SELFTEST    = { optional = true; tristate = whenOlder "4.18" "n";};
-      LNET_SELFTEST            = option (whenOlder "4.18" no);
+      LNET_SELFTEST            = { optional = true; tristate = whenOlder "4.18" "n";};
       LOCK_TORTURE_TEST        = option no;
       MTD_TESTS                = option no;
       NOTIFIER_ERROR_INJECTION = option no;
@@ -598,7 +592,7 @@ let
       AIC79XX_DEBUG_ENABLE = no;
       AIC7XXX_DEBUG_ENABLE = no;
       AIC94XX_DEBUG = no;
-      B43_PCMCIA = option (whenOlder "4.4" yes);
+      B43_PCMCIA = { optional=true; tristate = whenOlder "4.4" "y";};
 
       BLK_DEV_INTEGRITY       = yes;
 
@@ -651,7 +645,7 @@ let
       # GPIO on Intel Bay Trail, for some Chromebook internal eMMC disks
       PINCTRL_BAYTRAIL   = yes;
       # 8 is default. Modern gpt tables on eMMC may go far beyond 8.
-      MMC_BLOCK_MINORS   = "32";
+      MMC_BLOCK_MINORS   = freeform "32";
 
       REGULATOR  = yes; # Voltage and Current Regulator Support
       RC_DEVICES = option yes; # Enable IR devices
@@ -698,7 +692,8 @@ let
 
       # Bump the maximum number of CPUs to support systems like EC2 x1.*
       # instances and Xeon Phi.
-      NR_CPUS = "384";
+      NR_CPUS = freeform "384";
     };
   };
-in (generateNixKConf ((flattenKConf options) // structuredExtraConfig) mkValueOverride) + extraConfig
+in
+  flattenKConf options

pkgs/os-specific/linux/kernel/generic.nix

@@ -47,7 +47,6 @@
 , preferBuiltin ? stdenv.hostPlatform.platform.kernelPreferBuiltin or false
 , kernelArch ? stdenv.hostPlatform.platform.kernelArch
 
-, mkValueOverride ? null
 , ...
 }:
 
@@ -68,20 +67,26 @@ let
     ia32Emulation = true;
   } // features) kernelPatches;
 
-  intermediateNixConfig = import ./common-config.nix {
+  commonStructuredConfig = import ./common-config.nix {
-    inherit stdenv version structuredExtraConfig mkValueOverride;
+    inherit stdenv version ;
-
-    # append extraConfig for backwards compatibility but also means the user can't override the kernelExtraConfig part
-    extraConfig = extraConfig + lib.optionalString (stdenv.hostPlatform.platform ? kernelExtraConfig) stdenv.hostPlatform.platform.kernelExtraConfig;
 
     features = kernelFeatures; # Ensure we know of all extra patches, etc.
   };
 
-  kernelConfigFun = baseConfig:
+  # extra config in legacy string format
+  extraConfig = extraConfig + lib.optionalString (stdenv.hostPlatform.platform ? kernelExtraConfig) stdenv.hostPlatform.platform.kernelExtraConfig;
+
+  intermediateNixConfig = configfile.moduleStructuredConfig.intermediateNixConfig;
+
+  structuredConfigFromPatches =
+        map ({extraStructuredConfig ? {}, ...}: {settings=extraStructuredConfig;}) kernelPatches;
+
+  # appends kernel patches extraConfig
+  kernelConfigFun = baseConfigStr:
     let
       configFromPatches =
         map ({extraConfig ? "", ...}: extraConfig) kernelPatches;
-    in lib.concatStringsSep "\n" ([baseConfig] ++ configFromPatches);
+    in lib.concatStringsSep "\n" ([baseConfigStr] ++ configFromPatches);
 
   configfile = stdenv.mkDerivation {
     inherit ignoreConfigErrors autoModules preferBuiltin kernelArch;
@@ -131,7 +136,30 @@ let
     installPhase = "mv $buildRoot/.config $out";
 
     enableParallelBuilding = true;
-  };
+
+    passthru = rec {
+
+      module = import ../../../../nixos/modules/system/boot/kernel_config.nix;
+      # used also in apache
+      # { modules = [ { options = res.options; config = svc.config or svc; } ];
+      #   check = false;
+      # The result is a set of two attributes
+      moduleStructuredConfig = (lib.evalModules {
+        modules = [
+          module
+          { settings = commonStructuredConfig; }
+          { settings = structuredExtraConfig; }
+        ]
+        ++  structuredConfigFromPatches
+        ;
+      }).config;
+
+      #
+      structuredConfig = moduleStructuredConfig.settings;
+    };
+
+
+  }; # end of configfile derivation
 
   kernel = (callPackage ./manual-config.nix {}) {
     inherit version modDirVersion src kernelPatches stdenv extraMeta configfile;
@@ -141,6 +169,7 @@ let
 
   passthru = {
     features = kernelFeatures;
+    inherit commonStructuredConfig;
     passthru = kernel.passthru // (removeAttrs passthru [ "passthru" ]);
   };