From 28d2af4ea69fbc7f1803de4a3ab1e6dd49dfe519 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Jun 2024 09:28:27 -0400 Subject: [PATCH] Build `nix-util` with Meson MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idea is two-fold: - Replace autotools with Meson - Build each library in its own derivation The interaction of these two features is that Meson's "subprojects" feature (https://mesonbuild.com/Subprojects) allows us to have single dev shell for building all libraries still, while also building things separately. This allows us to break up the build without a huge productivity lost. I tested the Linux native build, and NetBSD and Windows cross builds. Also do some clean ups of the Flake in the process of supporting new jobs. Special thanks to everyone that has worked on a Meson port so far, @p01arst0rm and @Qyriad in particular. Co-Authored-By: p01arst0rm Co-Authored-By: Artemis Tosini Co-Authored-By: Artemis Tosini Co-Authored-By: Felix Uhl Co-Authored-By: Jade Lovelace Co-Authored-By: Lunaphied Co-Authored-By: Maximilian Bosch Co-Authored-By: Pierre Bourdon Co-Authored-By: Qyriad Co-Authored-By: Rebecca Turner Co-Authored-By: Winter Co-Authored-By: eldritch horrors Co-Authored-By: jade Co-Authored-By: julia Co-Authored-By: rebecca “wiggles” turner Co-Authored-By: wiggles dog Co-Authored-By: fricklerhandwerk Co-authored-By: Eli Schwartz Co-authored-by: Robert Hensing --- .gitignore | 2 + flake.nix | 75 +++++--- {build => maintainers}/hydra.nix | 12 +- meson.build | 9 + package.nix | 3 +- src/libutil/.version | 1 + src/libutil/linux/meson.build | 11 ++ src/libutil/meson.build | 288 +++++++++++++++++++++++++++++++ src/libutil/meson.options | 5 + src/libutil/package.nix | 144 ++++++++++++++++ src/libutil/unix/meson.build | 17 ++ src/libutil/windows/meson.build | 19 ++ 12 files changed, 553 insertions(+), 33 deletions(-) rename {build => maintainers}/hydra.nix (93%) create mode 100644 meson.build create mode 120000 src/libutil/.version create mode 100644 src/libutil/linux/meson.build create mode 100644 src/libutil/meson.build create mode 100644 src/libutil/meson.options create mode 100644 src/libutil/package.nix create mode 100644 src/libutil/unix/meson.build create mode 100644 src/libutil/windows/meson.build diff --git a/.gitignore b/.gitignore index 28f853715..1bf540ba2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ perl/Makefile.config /svn-revision /libtool /config/config.* +# Default meson build dir +/build # /doc/manual/ /doc/manual/*.1 diff --git a/flake.nix b/flake.nix index 5dbc554fc..0e29c6e6f 100644 --- a/flake.nix +++ b/flake.nix @@ -160,21 +160,23 @@ }; }); - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; + nix-util = final.callPackage ./src/libutil/package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + }; - in final.callPackage ./package.nix { + nix = + final.callPackage ./package.nix { inherit fileset stdenv + officialRelease versionSuffix ; - officialRelease = false; boehmgc = final.boehmgc-nix; libgit2 = final.libgit2-nix; libseccomp = final.libseccomp-nix; @@ -203,7 +205,7 @@ # 'nix.perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = import ./build/hydra.nix { + hydraJobs = import ./maintainers/hydra.nix { inherit inputs binaryTarball @@ -236,11 +238,29 @@ } // devFlake.checks.${system} or {} ); - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + packages = forAllSystems (system: { + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + } // lib.concatMapAttrs + (pkgName: {}: { + "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; + } // lib.concatMapAttrs + (crossSystem: {}: { + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}; + }) + (lib.genAttrs crossSystems (_: { })) + // lib.concatMapAttrs + (stdenvName: {}: { + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName}; + }) + (lib.genAttrs stdenvs (_: { }))) + { + "nix" = { }; + "nix-util" = { }; + } + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -255,18 +275,7 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: @@ -274,6 +283,11 @@ modular = devFlake.getSystem stdenv.buildPlatform.system; in { pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -288,12 +302,19 @@ src = null; env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.boost}/lib"; # For `make format`, to work without installing pre-commit _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; }; + inherit (pkgs.nix-util) mesonFlags; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nix-util.nativeBuildInputs ++ [ modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" diff --git a/build/hydra.nix b/maintainers/hydra.nix similarity index 93% rename from build/hydra.nix rename to maintainers/hydra.nix index 857b7f1f0..2541b061e 100644 --- a/build/hydra.nix +++ b/maintainers/hydra.nix @@ -32,6 +32,8 @@ let doBuild = false; }; + + forAllPackages = lib.genAttrs [ "nix" "nix-util" ]; in { # Binary package for various platforms. @@ -39,10 +41,12 @@ in shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName})); - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}")); + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } @@ -76,7 +80,7 @@ in binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: forAllCrossSystems (crossSystem: binaryTarball - self.packages.${system}."nix-${crossSystem}" + nixpkgsFor.${system}.cross.${crossSystem}.nix nixpkgsFor.${system}.cross.${crossSystem})); # The first half of the installation script. This is uploaded diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..fc6441877 --- /dev/null +++ b/meson.build @@ -0,0 +1,9 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +subproject('libutil') diff --git a/package.nix b/package.nix index cf1654c6a..677ee73c7 100644 --- a/package.nix +++ b/package.nix @@ -385,8 +385,7 @@ in { separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/src/libutil/.version b/src/libutil/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil/linux/meson.build b/src/libutil/linux/meson.build new file mode 100644 index 000000000..a1ded76ca --- /dev/null +++ b/src/libutil/linux/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'cgroup.cc', + 'namespaces.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'cgroup.hh', + 'namespaces.hh', +) diff --git a/src/libutil/meson.build b/src/libutil/meson.build new file mode 100644 index 000000000..8644c69f4 --- /dev/null +++ b/src/libutil/meson.build @@ -0,0 +1,288 @@ +project('nix-util', 'cpp', + version : run_command('cat', './.version', check : true).stdout().strip(), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 0.64.0', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +deps_private = [ ] +deps_public = [ ] +deps_other = [ ] + +configdata = configuration_data() + +# Check for each of these functions, and create a define like `#define HAVE_LUTIMES 1`. +check_funcs = [ + # Optionally used for changing the mtime of symlinks. + 'lutimes', + 'pipe2', + 'posix_fallocate', + 'strsignal', + 'sysconf', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +# Conditional to work around https://github.com/mesonbuild/meson/issues/13293 +if host_machine.system() != 'windows' and cxx.get_id() == 'gcc' + deps_private += dependency('threads') +endif + +if host_machine.system() == 'windows' + socket = cxx.find_library('ws2_32') + deps_other += socket +elif host_machine.system() == 'sunos' + socket = cxx.find_library('socket') + network_service_library = cxx.find_library('nsl') + deps_other += [socket, network_service_library] +endif + +boost = dependency( + 'boost', + modules : ['context', 'coroutine'], +) +# Actually public, but wrong type of dep for pkg config +deps_other += boost + +openssl = dependency( + 'libcrypto', + 'openssl', + version : '>= 1.1.1', +) +deps_private += openssl + +libarchive = dependency('libarchive', version : '>= 3.1.2') +deps_public += libarchive +if get_option('default_library') == 'static' + # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed + add_project_arguments('-lz', language : 'cpp') +endif + +sodium = dependency('libsodium', 'sodium') +deps_private += sodium + +brotli = [ + dependency('libbrotlicommon'), + dependency('libbrotlidec'), + dependency('libbrotlienc'), +] +deps_private += brotli + +# cpuid only makes sense on x86_64 +cpuid_required = host_machine.cpu_family() == 'x86_64' ? get_option('cpuid') : false +cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) +configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +config_util_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'archive.cc', + 'args.cc', + 'canon-path.cc', + 'compression.cc', + 'compute-levels.cc', + 'config.cc', + 'current-process.cc', + 'english.cc', + 'environment-variables.cc', + 'error.cc', + 'exit.cc', + 'experimental-features.cc', + 'file-content-address.cc', + 'file-descriptor.cc', + 'file-system.cc', + 'fs-sink.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'memory-source-accessor.cc', + 'position.cc', + 'posix-source-accessor.cc', + 'references.cc', + 'serialise.cc', + 'signature/local-keys.cc', + 'signature/signer.cc', + 'source-accessor.cc', + 'source-path.cc', + 'suggestions.cc', + 'tarfile.cc', + 'terminal.cc', + 'thread-pool.cc', + 'unix-domain-socket.cc', + 'url.cc', + 'users.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_util_h] + files( + 'abstract-setting-to-json.hh', + 'ansicolor.hh', + 'archive.hh', + 'args.hh', + 'args/root.hh', + 'callback.hh', + 'canon-path.hh', + 'chunked-vector.hh', + 'closure.hh', + 'comparator.hh', + 'compression.hh', + 'compute-levels.hh', + 'config-impl.hh', + 'config.hh', + 'current-process.hh', + 'english.hh', + 'environment-variables.hh', + 'error.hh', + 'exit.hh', + 'experimental-features.hh', + 'file-content-address.hh', + 'file-descriptor.hh', + 'file-path-impl.hh', + 'file-path.hh', + 'file-system.hh', + 'finally.hh', + 'fmt.hh', + 'fs-sink.hh', + 'git.hh', + 'hash.hh', + 'hilite.hh', + 'json-impls.hh', + 'json-utils.hh', + 'logging.hh', + 'lru-cache.hh', + 'memory-source-accessor.hh', + 'muxable-pipe.hh', + 'pool.hh', + 'position.hh', + 'posix-source-accessor.hh', + 'processes.hh', + 'ref.hh', + 'references.hh', + 'regex-combinators.hh', + 'repair-flag.hh', + 'serialise.hh', + 'signals.hh', + 'signature/local-keys.hh', + 'signature/signer.hh', + 'source-accessor.hh', + 'source-path.hh', + 'split.hh', + 'suggestions.hh', + 'sync.hh', + 'tarfile.hh', + 'terminal.hh', + 'thread-pool.hh', + 'topo-sort.hh', + 'types.hh', + 'unix-domain-socket.hh', + 'url-parts.hh', + 'url.hh', + 'users.hh', + 'util.hh', + 'variant-wrapper.hh', + 'xml-writer.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +libutil = library( + 'nixutil', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = ['-lboost_context', '-lboost_coroutine'] +if host_machine.system() == 'windows' + # `libraries_private` cannot contain ad-hoc dependencies (from + # `find_library), so we need to do this manually + libraries_private += ['-lws2_32'] +endif + +import('pkgconfig').generate( + libutil, + filebase : 'nix-util', + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : deps_public, + requires_private : deps_private, + # avoid absolute paths, use vendored ones + libraries_private : libraries_private, +) + +nix_util = declare_dependency( + include_directories : include_dirs, + link_with : libutil, + compile_args : ['-std=c++2a'], + dependencies : [], +) +meson.override_dependency('nix-util', nix_util) diff --git a/src/libutil/meson.options b/src/libutil/meson.options new file mode 100644 index 000000000..21883af01 --- /dev/null +++ b/src/libutil/meson.options @@ -0,0 +1,5 @@ +# vim: filetype=meson + +option('cpuid', type : 'feature', + description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', +) diff --git a/src/libutil/package.nix b/src/libutil/package.nix new file mode 100644 index 000000000..6575b37a6 --- /dev/null +++ b/src/libutil/package.nix @@ -0,0 +1,144 @@ +{ lib +, stdenv +, releaseTools +, fileset + +, meson +, ninja +, pkg-config + +, boost +, brotli +, libarchive +, libcpuid +, libsodium +, nlohmann_json +, openssl + +# Configuration Options + +, versionSuffix ? "" +, officialRelease ? false + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + boost + brotli + libsodium + openssl + ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ; + + propagatedBuildInputs = [ + libarchive + nlohmann_json + ]; + + disallowedReferences = [ boost ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + '' + # Copy libboost_context so we don't get all of Boost in our closure. + + # https://github.com/NixOS/nixpkgs/issues/45462 + + lib.optionalString (!stdenv.hostPlatform.isStatic) ('' + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + '' + lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + '' + ); + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + postInstall = + # Remove absolute path to boost libs + '' + sed -i "$out/lib/pkgconfig/nix-util.pc" -e 's, ${lib.getLib boost}[^ ]*,,g' + '' + + lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build new file mode 100644 index 000000000..38e5cd3aa --- /dev/null +++ b/src/libutil/unix/meson.build @@ -0,0 +1,17 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'processes.cc', + 'signals.cc', + 'users.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'monitor-fd.hh', + 'signals-impl.hh', +) diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build new file mode 100644 index 000000000..00320877f --- /dev/null +++ b/src/libutil/windows/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'processes.cc', + 'users.cc', + 'windows-async-pipe.cc', + 'windows-error.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'signals-impl.hh', + 'windows-async-pipe.hh', + 'windows-error.hh', +)