Merge pull request #10955 from NixOS/meson-nix-util-c

Build nix-util-c with meson + unit tests

flake.nix

@@ -58,6 +58,18 @@
         "stdenv"
       ];
 
+      /**
+        `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and
+        merges the results into a single attribute set.
+
+        This can be nested to form a build matrix where all the attributes
+        generated by the innermost `f` are returned as is.
+        (Provided that the names are unique.)
+
+        See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs
+       */
+      flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs;
+
       forAllSystems = lib.genAttrs systems;
 
       forAllCrossSystems = lib.genAttrs crossSystems;
@@ -160,6 +172,19 @@
             };
           });
 
+          # TODO: define everything here instead of top level?
+          nix-components = {
+            inherit (final)
+              nix-util
+              nix-util-test-support
+              nix-util-test
+              nix-util-c
+              nix-store
+              nix-fetchers
+              nix-perl-bindings
+              ;
+          };
+
           nix-util = final.callPackage ./src/libutil/package.nix {
             inherit
               fileset
@@ -169,6 +194,30 @@
               ;
           };
 
+          nix-util-test-support = final.callPackage ./tests/unit/libutil-support/package.nix {
+            inherit
+              fileset
+              stdenv
+              versionSuffix
+              ;
+          };
+
+          nix-util-test = final.callPackage ./tests/unit/libutil/package.nix {
+            inherit
+              fileset
+              stdenv
+              versionSuffix
+              ;
+          };
+
+          nix-util-c = final.callPackage ./src/libutil-c/package.nix {
+            inherit
+              fileset
+              stdenv
+              versionSuffix
+              ;
+          };
+
           nix-store = final.callPackage ./src/libstore/package.nix {
             inherit
               fileset
@@ -281,7 +330,25 @@
         # the old build system is gone and we are back to one build
         # system, we should reenable this.
         #perlBindings = self.hydraJobs.perlBindings.${system};
-      } // devFlake.checks.${system} or {}
+      }
+      # Add "passthru" tests
+      // flatMapAttrs ({
+          "" = nixpkgsFor.${system}.native;
+        } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) {
+          # TODO: enable static builds for darwin, blocked on:
+          #       https://github.com/NixOS/nixpkgs/issues/320448
+          "static-" = nixpkgsFor.${system}.static;
+        })
+        (nixpkgsPrefix: nixpkgs: 
+          flatMapAttrs nixpkgs.nix-components
+            (pkgName: pkg:
+              flatMapAttrs pkg.tests or {}
+              (testName: test: {
+                "${nixpkgsPrefix}${pkgName}-${testName}" = test;
+              })
+            )
+        )
+      // devFlake.checks.${system} or {}
       );
 
       packages = forAllSystems (system: {

maintainers/hydra.nix

@@ -36,6 +36,9 @@ let
   forAllPackages = lib.genAttrs [
     "nix"
     "nix-util"
+    "nix-util-c"
+    "nix-util-test-support"
+    "nix-util-test"
     "nix-store"
     "nix-fetchers"
   ];

meson.build

@@ -12,3 +12,10 @@ subproject('libfetchers')
 subproject('perl')
 subproject('internal-api-docs')
 subproject('external-api-docs')
+
+# C wrappers
+subproject('libutil-c')
+
+# Testing
+subproject('libutil-test-support')
+subproject('libutil-test')

src/libutil-c/.version

@@ -0,0 +1 @@
+../../.version

src/libutil-c/meson.build

@@ -0,0 +1,117 @@
+project('nix-util-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+# See note in ../nix-util/meson.build
+deps_private = [ ]
+
+# See note in ../nix-util/meson.build
+deps_public = [ ]
+
+# See note in ../nix-util/meson.build
+deps_other = [ ]
+
+configdata = configuration_data()
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.h',
+  # '-include', 'config-store.h',
+  '-Wno-deprecated-declarations',
+  '-Wimplicit-fallthrough',
+  '-Werror=switch',
+  '-Werror=switch-enum',
+  '-Wdeprecated-copy',
+  '-Wignored-qualifiers',
+  # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
+  # at ~1% overhead in `nix search`.
+  #
+  # FIXME: remove when we get meson 1.4.0 which will default this to on for us:
+  # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions
+  '-D_GLIBCXX_ASSERTIONS=1',
+  language : 'cpp',
+)
+
+sources = files(
+  'nix_api_util.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = files(
+  'nix_api_util.h',
+  'nix_api_util_internal.h',
+)
+
+if host_machine.system() == 'cygwin' or host_machine.system() == 'windows'
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  linker_export_flags = ['-Wl,--export-all-symbols']
+else
+  linker_export_flags = []
+endif
+
+nix_util = dependency('nix-util')
+if nix_util.type_name() == 'internal'
+  # subproject sadly no good for pkg-config module
+  deps_other += nix_util
+else
+  deps_public += nix_util
+endif
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-util.h',
+)
+
+this_library = library(
+  'nixutilc',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+import('pkgconfig').generate(
+  this_library,
+  filebase : meson.project_name(),
+  name : 'Nix',
+  description : 'Nix Package Manager',
+  subdirs : ['nix'],
+  extra_cflags : ['-std=c++2a'],
+  requires : deps_public,
+  requires_private : deps_private,
+  libraries_private : libraries_private,
+)
+
+meson.override_dependency(meson.project_name(), declare_dependency(
+  include_directories : include_dirs,
+  link_with : this_library,
+  compile_args : ['-std=c++2a'],
+  dependencies : [],
+))

src/libutil-c/meson.options

@@ -0,0 +1 @@
+# vim: filetype=meson

src/libutil-c/nix-util-c.pc.in

@@ -0,0 +1,9 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix libutil C API
+Description: Common functions for the Nix C API, such as error handling
+Version: @PACKAGE_VERSION@
+Libs: -L${libdir} -lnixutil
+Cflags: -I${includedir}/nix -std=c++2a

src/libutil-c/package.nix

@@ -0,0 +1,96 @@
+{ lib
+, stdenv
+, releaseTools
+, fileset
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+
+# Configuration Options
+
+, versionSuffix ? ""
+
+# Check test coverage of Nix. Probably want to use with at least
+# one of `doCheck` or `doInstallCheck` enabled.
+, withCoverageChecks ? false
+}:
+
+let
+  version = lib.fileContents ./.version + versionSuffix;
+
+  mkDerivation =
+    if withCoverageChecks
+    then
+      # TODO support `finalAttrs` args function in
+      # `releaseTools.coverageAnalysis`.
+      argsFun:
+         releaseTools.coverageAnalysis (let args = argsFun args; in args)
+    else stdenv.mkDerivation;
+in
+
+mkDerivation (finalAttrs: {
+  pname = "nix-util-c";
+  inherit version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions [
+      ./meson.build
+      ./meson.options
+      (fileset.fileFilter (file: file.hasExt "cc") ./.)
+      (fileset.fileFilter (file: file.hasExt "hh") ./.)
+      (fileset.fileFilter (file: file.hasExt "h") ./.)
+    ];
+  };
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    nix-util
+  ]
+  ;
+
+  propagatedBuildInputs = [
+    nix-util
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix
+    ''
+      echo ${version} > .version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
+  strictDeps = !withCoverageChecks;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+} // lib.optionalAttrs withCoverageChecks {
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = [ "fortify" ];
+})

src/libutil-test

@@ -0,0 +1 @@
+../tests/unit/libutil/

src/libutil-test-support

@@ -0,0 +1 @@
+../tests/unit/libutil-support/

tests/unit/libutil-support/.version

@@ -0,0 +1 @@
+../../../.version

tests/unit/libutil-support/meson.build

@@ -0,0 +1,110 @@
+project('nix-util-test-support', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+# See note in ../nix-util/meson.build
+deps_private = [ ]
+
+# See note in ../nix-util/meson.build
+deps_public = [ ]
+
+# See note in ../nix-util/meson.build
+deps_other = [ ]
+
+add_project_arguments(
+  '-Wno-deprecated-declarations',
+  '-Wimplicit-fallthrough',
+  '-Werror=switch',
+  '-Werror=switch-enum',
+  '-Wdeprecated-copy',
+  '-Wignored-qualifiers',
+  # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
+  # at ~1% overhead in `nix search`.
+  #
+  # FIXME: remove when we get meson 1.4.0 which will default this to on for us:
+  # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions
+  '-D_GLIBCXX_ASSERTIONS=1',
+  language : 'cpp',
+)
+
+sources = files(
+  'tests/hash.cc',
+  'tests/string_callback.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = files(
+  'tests/characterization.hh',
+  'tests/hash.hh',
+  'tests/nix_api_util.hh',
+  'tests/string_callback.hh',
+)
+
+if host_machine.system() == 'cygwin' or host_machine.system() == 'windows'
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  linker_export_flags = ['-Wl,--export-all-symbols']
+else
+  linker_export_flags = []
+endif
+
+nix_util = dependency('nix-util')
+if nix_util.type_name() == 'internal'
+  # subproject sadly no good for pkg-config module
+  deps_other += nix_util
+else
+  deps_public += nix_util
+endif
+
+rapidcheck = dependency('rapidcheck')
+deps_public += rapidcheck
+
+this_library = library(
+  'nix-util-test-support',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326
+  #       is available. See also ../libutil/build.meson
+  link_args: linker_export_flags + ['-lrapidcheck'],
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+import('pkgconfig').generate(
+  this_library,
+  filebase : meson.project_name(),
+  name : 'Nix',
+  description : 'Nix Package Manager',
+  subdirs : ['nix'],
+  extra_cflags : ['-std=c++2a'],
+  requires : deps_public,
+  requires_private : deps_private,
+)
+
+meson.override_dependency(meson.project_name(), declare_dependency(
+  include_directories : include_dirs,
+  link_with : this_library,
+  compile_args : ['-std=c++2a'],
+  dependencies : [],
+))

tests/unit/libutil-support/package.nix

@@ -0,0 +1,98 @@
+{ lib
+, stdenv
+, releaseTools
+, fileset
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+
+, rapidcheck
+
+# Configuration Options
+
+, versionSuffix ? ""
+
+# Check test coverage of Nix. Probably want to use with at least
+# one of `doCheck` or `doInstallCheck` enabled.
+, withCoverageChecks ? false
+}:
+
+let
+  version = lib.fileContents ./.version + versionSuffix;
+
+  mkDerivation =
+    if withCoverageChecks
+    then
+      # TODO support `finalAttrs` args function in
+      # `releaseTools.coverageAnalysis`.
+      argsFun:
+         releaseTools.coverageAnalysis (let args = argsFun args; in args)
+    else stdenv.mkDerivation;
+in
+
+mkDerivation (finalAttrs: {
+  pname = "nix-util-test-support";
+  inherit version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions [
+      ./meson.build
+      # ./meson.options
+      (fileset.fileFilter (file: file.hasExt "cc") ./.)
+      (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    ];
+  };
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    nix-util
+    rapidcheck
+  ]
+  ;
+
+  propagatedBuildInputs = [
+    nix-util
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix
+    ''
+      echo ${version} > .version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
+  strictDeps = !withCoverageChecks;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+} // lib.optionalAttrs withCoverageChecks {
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = [ "fortify" ];
+})

tests/unit/libutil/.version

@@ -0,0 +1 @@
+2.24.0

tests/unit/libutil/meson.build

@@ -0,0 +1,142 @@
+project('nix-util-test', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+# See note in ../nix-util/meson.build
+deps_private = [ ]
+
+# See note in ../nix-util/meson.build
+deps_public = [ ]
+
+# See note in ../nix-util/meson.build
+deps_other = [ ]
+
+configdata = configuration_data()
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util-test.h',
+  # '-include', 'config-store.h',
+  '-Wno-deprecated-declarations',
+  '-Wimplicit-fallthrough',
+  '-Werror=switch',
+  '-Werror=switch-enum',
+  '-Wdeprecated-copy',
+  '-Wignored-qualifiers',
+  # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
+  # at ~1% overhead in `nix search`.
+  #
+  # FIXME: remove when we get meson 1.4.0 which will default this to on for us:
+  # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions
+  '-D_GLIBCXX_ASSERTIONS=1',
+  language : 'cpp',
+)
+
+# TODO rename, because it will conflict with downstream projects
+configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
+
+config_h = configure_file(
+  configuration : configdata,
+  output : 'config-util-test.h',
+)
+
+sources = files(
+  'args.cc',
+  'canon-path.cc',
+  'chunked-vector.cc',
+  'closure.cc',
+  'compression.cc',
+  'config.cc',
+  'file-content-address.cc',
+  'git.cc',
+  'hash.cc',
+  'hilite.cc',
+  'json-utils.cc',
+  'logging.cc',
+  'lru-cache.cc',
+  'nix_api_util.cc',
+  'pool.cc',
+  'references.cc',
+  'spawn.cc',
+  'suggestions.cc',
+  'tests.cc',
+  'url.cc',
+  'xml-writer.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+if host_machine.system() == 'cygwin' or host_machine.system() == 'windows'
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  linker_export_flags = ['-Wl,--export-all-symbols']
+else
+  linker_export_flags = []
+endif
+
+nix_util = dependency('nix-util')
+if nix_util.type_name() == 'internal'
+  # subproject sadly no good for pkg-config module
+  deps_other += nix_util
+else
+  deps_public += nix_util
+endif
+
+nix_util_c = dependency('nix-util-c')
+if nix_util_c.type_name() == 'internal'
+  # subproject sadly no good for pkg-config module
+  deps_other += nix_util_c
+else
+  deps_public += nix_util_c
+endif
+
+nix_util_test_support = dependency('nix-util-test-support')
+if nix_util_test_support.type_name() == 'internal'
+  # subproject sadly no good for pkg-config module
+  deps_other += nix_util_test_support
+else
+  deps_public += nix_util_test_support
+endif
+
+rapidcheck = dependency('rapidcheck')
+deps_public += rapidcheck
+
+gtest = dependency('gtest', main : true)
+deps_public += gtest
+
+this_exe = executable(
+  'nix-util-test',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  # TODO: -lrapidcheck, see ../libutil-support/build.meson
+  link_args: linker_export_flags + ['-lrapidcheck'],
+  # get main from gtest
+  install : true,
+)
+
+test('nix-util-test', this_exe, env : ['_NIX_TEST_UNIT_DATA=' + meson.current_source_dir() + '/data'])
+
+meson.override_dependency(meson.project_name(), declare_dependency(
+  include_directories : include_dirs,
+  link_with : this_exe,
+  compile_args : ['-std=c++2a'],
+  dependencies : [],
+))

tests/unit/libutil/package.nix

@@ -0,0 +1,112 @@
+{ lib
+, stdenv
+, releaseTools
+, fileset
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+, nix-util-c
+, nix-util-test-support
+
+, rapidcheck
+, gtest
+, runCommand
+
+# Configuration Options
+
+, versionSuffix ? ""
+
+# Check test coverage of Nix. Probably want to use with at least
+# one of `doCheck` or `doInstallCheck` enabled.
+, withCoverageChecks ? false
+}:
+
+let
+  version = lib.fileContents ./.version + versionSuffix;
+
+  mkDerivation =
+    if withCoverageChecks
+    then
+      # TODO support `finalAttrs` args function in
+      # `releaseTools.coverageAnalysis`.
+      argsFun:
+         releaseTools.coverageAnalysis (let args = argsFun args; in args)
+    else stdenv.mkDerivation;
+in
+
+mkDerivation (finalAttrs: {
+  pname = "nix-util-test";
+  inherit version;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = fileset.unions [
+      ./meson.build
+      # ./meson.options
+      (fileset.fileFilter (file: file.hasExt "cc") ./.)
+      (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    ];
+  };
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    nix-util
+    nix-util-c
+    nix-util-test-support
+    rapidcheck
+    gtest
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix
+    ''
+      echo ${version} > .version
+    '';
+
+  mesonFlags = [
+  ];
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564
+  strictDeps = !withCoverageChecks;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  passthru = {
+    tests = {
+      run = runCommand "${finalAttrs.pname}-run" {
+      } ''
+        PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH"
+        export _NIX_TEST_UNIT_DATA=${./data}
+        nix-util-test
+        touch $out
+      '';
+    };
+  };
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+} // lib.optionalAttrs withCoverageChecks {
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = [ "fortify" ];
+})