nordic-dev.net/nixpkgs
nordic-dev.net/nixpkgs
2
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2024-10-30 06:01:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #75584 from Infinisil/settings-formats

Browse Source 
Configuration file formats for JSON, INI, YAML and TOML
This commit is contained in:
Robert Hensing 2020-08-02 16:58:49 +02:00 committed by GitHub
commit 150bf4fa3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 489 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/generators.nix
View File

@ -48,8 +48,10 @@ rec {
else if isAttrs v then err "attrsets" v
# functions can’t be printed of course
else if isFunction v then err "functions" v
# let’s not talk about floats. There is no sensible `toString` for them.
else if isFloat v then err "floats" v
# Floats currently can't be converted to precise strings,
# condition warning on nix version once this isn't a problem anymore
# See https://github.com/NixOS/nix/pull/3480
else if isFloat v then libStr.floatToString v
else err "this value is" (toString v);

16
lib/strings.nix
View File

@ -612,6 +612,22 @@ rec {
*/
fixedWidthNumber = width: n: fixedWidthString width "0" (toString n);
/* Convert a float to a string, but emit a warning when precision is lost
during the conversion
Example:
floatToString 0.000001
=> "0.000001"
floatToString 0.0000001
=> trace: warning: Imprecise conversion from float to string 0.000000
"0.000000"
*/
floatToString = float: let
result = toString float;
precise = float == builtins.fromJSON result;
in if precise then result
else lib.warn "Imprecise conversion from float to string ${result}" result;
/* Check whether a value can be coerced to a string */
isCoercibleToString = x:
builtins.elem (builtins.typeOf x) [ "path" "string" "null" "int" "float" "bool" ] ||

179
nixos/doc/manual/development/settings-options.xml Normal file
View File

@ -0,0 +1,179 @@
<section xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
version="5.0"
xml:id="sec-settings-options">
<title>Options for Program Settings</title>
<para>
Many programs have configuration files where program-specific settings can be declared. File formats can be separated into two categories:
<itemizedlist>
<listitem>
<para>
Nix-representable ones: These can trivially be mapped to a subset of Nix syntax. E.g. JSON is an example, since its values like <literal>{"foo":{"bar":10}}</literal> can be mapped directly to Nix: <literal>{ foo = { bar = 10; }; }</literal>. Other examples are INI, YAML and TOML. The following section explains the convention for these settings.
</para>
</listitem>
<listitem>
<para>
Non-nix-representable ones: These can't be trivially mapped to a subset of Nix syntax. Most generic programming languages are in this group, e.g. bash, since the statement <literal>if true; then echo hi; fi</literal> doesn't have a trivial representation in Nix.
</para>
<para>
Currently there are no fixed conventions for these, but it is common to have a <literal>configFile</literal> option for setting the configuration file path directly. The default value of <literal>configFile</literal> can be an auto-generated file, with convenient options for controlling the contents. For example an option of type <literal>attrsOf str</literal> can be used for representing environment variables which generates a section like <literal>export FOO="foo"</literal>. Often it can also be useful to also include an <literal>extraConfig</literal> option of type <literal>lines</literal> to allow arbitrary text after the autogenerated part of the file.
</para>
</listitem>
</itemizedlist>
</para>
<section xml:id="sec-settings-nix-representable">
<title>Nix-representable Formats (JSON, YAML, TOML, INI, ...)</title>
<para>
By convention, formats like this are handled with a generic <literal>settings</literal> option, representing the full program configuration as a Nix value. The type of this option should represent the format. The most common formats have a predefined type and string generator already declared under <literal>pkgs.formats</literal>:
<variablelist>
<varlistentry>
<term>
<varname>pkgs.formats.json</varname> { }
</term>
<listitem>
<para>
A function taking an empty attribute set (for future extensibility) and returning a set with JSON-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<varname>pkgs.formats.yaml</varname> { }
</term>
<listitem>
<para>
A function taking an empty attribute set (for future extensibility) and returning a set with YAML-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<varname>pkgs.formats.ini</varname> { <replaceable>listsAsDuplicateKeys</replaceable> ? false, ... }
</term>
<listitem>
<para>
A function taking an attribute set with values
<variablelist>
<varlistentry>
<term>
<varname>listsAsDuplicateKeys</varname>
</term>
<listitem>
<para>
A boolean for controlling whether list values can be used to represent duplicate INI keys
</para>
</listitem>
</varlistentry>
</variablelist>
It returns a set with INI-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<varname>pkgs.formats.toml</varname> { }
</term>
<listitem>
<para>
A function taking an empty attribute set (for future extensibility) and returning a set with TOML-specific attributes <varname>type</varname> and <varname>generate</varname> as specified <link linkend='pkgs-formats-result'>below</link>.
</para>
</listitem>
</varlistentry>
</variablelist>
</para>
<para xml:id="pkgs-formats-result">
These functions all return an attribute set with these values:
<variablelist>
<varlistentry>
<term>
<varname>type</varname>
</term>
<listitem>
<para>
A module system type representing a value of the format
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<varname>generate</varname> <replaceable>filename</replaceable> <replaceable>jsonValue</replaceable>
</term>
<listitem>
<para>
A function that can render a value of the format to a file. Returns a file path.
<note>
<para>
This function puts the value contents in the Nix store. So this should be avoided for secrets.
</para>
</note>
</para>
</listitem>
</varlistentry>
</variablelist>
</para>
<example xml:id="ex-settings-nix-representable">
<title>Module with conventional <literal>settings</literal> option</title>
<para>
The following shows a module for an example program that uses a JSON configuration file. It demonstrates how above values can be used, along with some other related best practices. See the comments for explanations.
</para>
<programlisting>
{ options, config, lib, pkgs, ... }:
let
cfg = config.services.foo;
# Define the settings format used for this program
settingsFormat = pkgs.formats.json {};
in {
options.services.foo = {
enable = lib.mkEnableOption "foo service";
settings = lib.mkOption {
# Setting this type allows for correct merging behavior
type = settingsFormat.type;
default = {};
description = ''
Configuration for foo, see
&lt;link xlink:href="https://example.com/docs/foo"/&gt;
for supported values.
'';
};
};
config = lib.mkIf cfg.enable {
# We can assign some default settings here to make the service work by just
# enabling it. We use `mkDefault` for values that can be changed without
# problems
services.foo.settings = {
# Fails at runtime without any value set
log_level = lib.mkDefault "WARN";
# We assume systemd's `StateDirectory` is used, so we require this value,
# therefore no mkDefault
data_path = "/var/lib/foo";
# Since we use this to create a user we need to know the default value at
# eval time
user = lib.mkDefault "foo";
};
environment.etc."foo.json".source =
# The formats generator function takes a filename and the Nix value
# representing the format value and produces a filepath with that value
# rendered in the format
settingsFormat.generate "foo-config.json" cfg.settings;
# We know that the `user` attribute exists because we set a default value
# for it above, allowing us to use it without worries here
users.users.${cfg.settings.user} = {}
# ...
};
}
</programlisting>
</example>
</section>
</section>

1
nixos/doc/manual/development/writing-modules.xml
View File

@ -183,4 +183,5 @@ in {
<xi:include href="meta-attributes.xml" />
<xi:include href="importing-modules.xml" />
<xi:include href="replace-modules.xml" />
<xi:include href="settings-options.xml" />
</chapter>

11
pkgs/pkgs-lib/default.nix Normal file
View File

@ -0,0 +1,11 @@
# pkgs-lib is for functions and values that can't be in lib because
# they depend on some packages. This notably is *not* for supporting package
# building, instead pkgs/build-support is the place for that.
{ lib, pkgs }: {
# setting format types and generators. These do not fit in lib/types.nix,
# because they depend on pkgs for rendering some formats
formats = import ./formats.nix {
inherit lib pkgs;
};
}

109
pkgs/pkgs-lib/formats.nix Normal file
View File

@ -0,0 +1,109 @@
{ lib, pkgs }:
rec {
/*
Every following entry represents a format for program configuration files
used for `settings`-style options (see https://github.com/NixOS/rfcs/pull/42).
Each entry should look as follows:
<format> = <parameters>: {
# ^^ Parameters for controlling the format
# The module system type most suitable for representing such a format
# The description needs to be overwritten for recursive types
type = ...;
# generate :: Name -> Value -> Path
# A function for generating a file with a value of such a type
generate = ...;
});
*/
json = {}: {
type = with lib.types; let
valueType = nullOr (oneOf [
bool
int
float
str
(attrsOf valueType)
(listOf valueType)
]) // {
description = "JSON value";
};
in valueType;
generate = name: value: pkgs.runCommandNoCC name {
nativeBuildInputs = [ pkgs.jq ];
value = builtins.toJSON value;
passAsFile = [ "value" ];
} ''
jq . "$valuePath"> $out
'';
};
# YAML has been a strict superset of JSON since 1.2
yaml = {}:
let jsonSet = json {};
in jsonSet // {
type = jsonSet.type // {
description = "YAML value";
};
};
ini = { listsAsDuplicateKeys ? false, ... }@args: {
type = with lib.types; let
singleIniAtom = nullOr (oneOf [
bool
int
float
str
]) // {
description = "INI atom (null, bool, int, float or string)";
};
iniAtom =
if listsAsDuplicateKeys then
coercedTo singleIniAtom lib.singleton (listOf singleIniAtom) // {
description = singleIniAtom.description + " or a list of them for duplicate keys";
}
else
singleIniAtom;
in attrsOf (attrsOf iniAtom);
generate = name: value: pkgs.writeText name (lib.generators.toINI args value);
};
toml = {}: json {} // {
type = with lib.types; let
valueType = oneOf [
bool
int
float
str
(attrsOf valueType)
(listOf valueType)
] // {
description = "TOML value";
};
in valueType;
generate = name: value: pkgs.runCommandNoCC name {
nativeBuildInputs = [ pkgs.remarshal ];
value = builtins.toJSON value;
passAsFile = [ "value" ];
} ''
json2toml "$valuePath" "$out"
'';
};
}

7
pkgs/pkgs-lib/tests/default.nix Normal file
View File

@ -0,0 +1,7 @@
# Call nix-build on this file to run all tests in this directory
{ pkgs ? import ../../.. {} }:
let
formats = import ./formats.nix { inherit pkgs; };
in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [
{ name = "formats"; path = import ./formats.nix { inherit pkgs; }; }
]

157
pkgs/pkgs-lib/tests/formats.nix Normal file
View File

@ -0,0 +1,157 @@
{ pkgs }:
let
inherit (pkgs) lib formats;
in
with lib;
let
evalFormat = format: args: def:
let
formatSet = format args;
config = formatSet.type.merge [] (imap1 (n: def: {
value = def;
file = "def${toString n}";
}) [ def ]);
in formatSet.generate "test-format-file" config;
runBuildTest = name: { drv, expected }: pkgs.runCommandNoCC name {} ''
if diff ${drv} ${builtins.toFile "expected" expected}; then
touch $out
else
echo "Got: $(cat ${drv})"
echo "Should be: ${expected}"
exit 1
fi
'';
runBuildTests = tests: pkgs.linkFarm "nixpkgs-pkgs-lib-format-tests" (mapAttrsToList (name: value: { inherit name; path = runBuildTest name value; }) (filterAttrs (name: value: value != null) tests));
in runBuildTests {
testJsonAtoms = {
drv = evalFormat formats.json {} {
null = null;
false = false;
true = true;
int = 10;
float = 3.141;
str = "foo";
attrs.foo = null;
list = [ null null ];
};
expected = ''
{
"attrs": {
"foo": null
},
"false": false,
"float": 3.141,
"int": 10,
"list": [
null,
null
],
"null": null,
"str": "foo",
"true": true
}
'';
};
testYamlAtoms = {
drv = evalFormat formats.yaml {} {
null = null;
false = false;
true = true;
float = 3.141;
str = "foo";
attrs.foo = null;
list = [ null null ];
};
expected = ''
{
"attrs": {
"foo": null
},
"false": false,
"float": 3.141,
"list": [
null,
null
],
"null": null,
"str": "foo",
"true": true
}
'';
};
testIniAtoms = {
drv = evalFormat formats.ini {} {
foo = {
bool = true;
int = 10;
float = 3.141;
str = "string";
};
};
expected = ''
[foo]
bool=true
float=3.141000
int=10
str=string
'';
};
testIniDuplicateKeys = {
drv = evalFormat formats.ini { listsAsDuplicateKeys = true; } {
foo = {
bar = [ null true "test" 1.2 10 ];
baz = false;
qux = "qux";
};
};
expected = ''
[foo]
bar=null
bar=true
bar=test
bar=1.200000
bar=10
baz=false
qux=qux
'';
};
testTomlAtoms = {
drv = evalFormat formats.toml {} {
false = false;
true = true;
int = 10;
float = 3.141;
str = "foo";
attrs.foo = "foo";
list = [ 1 2 ];
level1.level2.level3.level4 = "deep";
};
expected = ''
false = false
float = 3.141
int = 10
list = [1, 2]
str = "foo"
true = true
[attrs]
foo = "foo"
[level1]
[level1.level2]
[level1.level2.level3]
level4 = "deep"
'';
};
}

3
pkgs/top-level/all-packages.nix
View File

@ -525,6 +525,9 @@ in
#package writers
writers = callPackage ../build-support/writers {};
# lib functions depending on pkgs
inherit (import ../pkgs-lib { inherit lib pkgs; }) formats;
### TOOLS
_0x0 = callPackage ../tools/misc/0x0 { };

2
pkgs/top-level/release.nix
View File

@ -34,6 +34,7 @@ let
manual = import ../../doc { inherit pkgs nixpkgs; };
lib-tests = import ../../lib/tests/release.nix { inherit pkgs; };
pkgs-lib-tests = import ../pkgs-lib/tests { inherit pkgs; };
darwin-tested = if supportDarwin then pkgs.releaseTools.aggregate
{ name = "nixpkgs-darwin-${jobs.tarball.version}";
@ -91,6 +92,7 @@ let
[ jobs.tarball
jobs.manual
jobs.lib-tests
jobs.pkgs-lib-tests
jobs.stdenv.x86_64-linux
jobs.linux.x86_64-linux
jobs.pandoc.x86_64-linux