From 992b656c9ea6ab583123273f39f1b22a8f29e96d Mon Sep 17 00:00:00 2001 From: IvarWithoutBones Date: Thu, 13 Jan 2022 07:39:44 +0100 Subject: [PATCH 1/2] buildDotnetModule: use setup hooks This is a much more flexible way of doing things, as we can adopt and reuse these hooks for various tasks. Syntax highlighting now also works way better for me, which is a nice bonus :) --- .../dotnet/build-dotnet-module/default.nix | 209 +++++------------- .../build-dotnet-module/hooks/default.nix | 62 ++++++ .../hooks/dotnet-build-hook.sh | 41 ++++ .../hooks/dotnet-check-hook.sh | 33 +++ .../hooks/dotnet-configure-hook.sh | 32 +++ .../hooks/dotnet-fixup-hook.sh | 42 ++++ .../hooks/dotnet-install-hook.sh | 42 ++++ 7 files changed, 307 insertions(+), 154 deletions(-) create mode 100644 pkgs/build-support/dotnet/build-dotnet-module/hooks/default.nix create mode 100644 pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh create mode 100644 pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-check-hook.sh create mode 100644 pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh create mode 100644 pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh create mode 100644 pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index 49a61f4e5d6d..e721b59decbe 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -1,6 +1,7 @@ -{ lib, stdenvNoCC, linkFarmFromDrvs, nuget-to-nix, writeScript, makeWrapper, fetchurl, xml2, dotnetCorePackages, dotnetPackages, cacert }: +{ lib, stdenvNoCC, linkFarmFromDrvs, callPackage, nuget-to-nix, writeScript, makeWrapper, fetchurl, xml2, dotnetCorePackages, dotnetPackages, cacert }: { name ? "${args.pname}-${args.version}" +, pname ? name , enableParallelBuilding ? true , doCheck ? false # Flags to pass to `makeWrapper`. This is done to avoid double wrapping. @@ -19,6 +20,8 @@ # Flags to pass to dotnet in all phases. , dotnetFlags ? [] +# The path to publish the project to. When unset, the directory "$out/lib/$pname" is used. +, installPath ? null # The binaries that should get installed to `$out/bin`, relative to `$out/lib/$pname/`. These get wrapped accordingly. # Unfortunately, dotnet has no method for doing this automatically. # If unset, all executables in the projects root will get installed. This may cause bloat! @@ -67,6 +70,10 @@ assert projectFile == null -> throw "Defining the `projectFile` attribute is req assert nugetDeps == null -> throw "Defining the `nugetDeps` attribute is required, as to lock the NuGet dependencies. This file can be generated by running the `passthru.fetch-deps` script."; let + inherit (callPackage ./hooks { + inherit dotnet-sdk dotnet-test-sdk disabledTests nuget-source dotnet-runtime runtimeDeps buildType; + }) dotnetConfigureHook dotnetBuildHook dotnetCheckHook dotnetInstallHook dotnetFixupHook; + _nugetDeps = linkFarmFromDrvs "${name}-nuget-deps" (import nugetDeps { fetchNuGet = { pname, version, sha256 }: fetchurl { name = "${pname}-${version}.nupkg"; @@ -77,8 +84,8 @@ let _localDeps = linkFarmFromDrvs "${name}-local-nuget-deps" projectReferences; nuget-source = stdenvNoCC.mkDerivation rec { - name = "${args.pname}-nuget-source"; - meta.description = "A Nuget source with the dependencies for ${args.pname}"; + name = "${pname}-nuget-source"; + meta.description = "A Nuget source with the dependencies for ${pname}"; nativeBuildInputs = [ dotnetPackages.Nuget xml2 ]; buildCommand = '' @@ -103,168 +110,62 @@ let ))); }; - package = stdenvNoCC.mkDerivation (args // { - inherit buildType; +in stdenvNoCC.mkDerivation (args // { + nativeBuildInputs = args.nativeBuildInputs or [] ++ [ + dotnetConfigureHook + dotnetBuildHook + dotnetCheckHook + dotnetInstallHook + dotnetFixupHook - nativeBuildInputs = args.nativeBuildInputs or [] ++ [ dotnet-sdk cacert makeWrapper ]; + dotnet-sdk + cacert + makeWrapper + ]; - # Stripping breaks the executable - dontStrip = true; + # Stripping breaks the executable + dontStrip = args.dontStrip or true; - # gappsWrapperArgs gets included when wrapping for dotnet, as to avoid double wrapping - dontWrapGApps = true; + # gappsWrapperArgs gets included when wrapping for dotnet, as to avoid double wrapping + dontWrapGApps = args.dontWrapGApps or true; - DOTNET_NOLOGO = true; # This disables the welcome message. - DOTNET_CLI_TELEMETRY_OPTOUT = true; + DOTNET_NOLOGO = args.DOTNET_NOLOGO or true; # This disables the welcome message. + DOTNET_CLI_TELEMETRY_OPTOUT = args.DOTNET_CLI_TELEMETRY_OPTOUT or true; - passthru = { - fetch-deps = writeScript "fetch-${args.pname}-deps" '' - set -euo pipefail - cd "$(dirname "''${BASH_SOURCE[0]}")" - - export HOME=$(mktemp -d) - deps_file="/tmp/${args.pname}-deps.nix" - - store_src="${package.src}" - src="$(mktemp -d /tmp/${args.pname}.XXX)" - cp -rT "$store_src" "$src" - chmod -R +w "$src" - - trap "rm -rf $src $HOME" EXIT - pushd "$src" - - export DOTNET_NOLOGO=1 - export DOTNET_CLI_TELEMETRY_OPTOUT=1 - - mkdir -p "$HOME/nuget_pkgs" - - for project in "${lib.concatStringsSep "\" \"" ((lib.toList projectFile) ++ lib.optionals (testProjectFile != "") (lib.toList testProjectFile))}"; do - ${dotnet-sdk}/bin/dotnet restore "$project" \ - ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \ - -p:ContinuousIntegrationBuild=true \ - -p:Deterministic=true \ - --packages "$HOME/nuget_pkgs" \ - "''${dotnetRestoreFlags[@]}" \ - "''${dotnetFlags[@]}" - done - - echo "Writing lockfile..." - ${nuget-to-nix}/bin/nuget-to-nix "$HOME/nuget_pkgs" > "$deps_file" - echo "Succesfully wrote lockfile to: $deps_file" - ''; - } // args.passthru or {}; - - configurePhase = args.configurePhase or '' - runHook preConfigure + passthru = { + fetch-deps = writeScript "fetch-${pname}-deps" '' + set -euo pipefail + cd "$(dirname "''${BASH_SOURCE[0]}")" export HOME=$(mktemp -d) + deps_file="/tmp/${pname}-deps.nix" - for project in ''${projectFile[@]} ''${testProjectFile[@]}; do - dotnet restore "$project" \ + store_src="${args.src}" + src="$(mktemp -d /tmp/${pname}.XXX)" + cp -rT "$store_src" "$src" + chmod -R +w "$src" + + trap "rm -rf $src $HOME" EXIT + pushd "$src" + + export DOTNET_NOLOGO=1 + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + + mkdir -p "$HOME/nuget_pkgs" + + for project in "${lib.concatStringsSep "\" \"" ((lib.toList projectFile) ++ lib.optionals (testProjectFile != "") (lib.toList testProjectFile))}"; do + ${dotnet-sdk}/bin/dotnet restore "$project" \ ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \ -p:ContinuousIntegrationBuild=true \ -p:Deterministic=true \ - --source "${nuget-source}/lib" \ - "''${dotnetRestoreFlags[@]}" \ - "''${dotnetFlags[@]}" + --packages "$HOME/nuget_pkgs" \ + ${lib.optionalString (dotnetRestoreFlags != []) (builtins.toString dotnetRestoreFlags)} \ + ${lib.optionalString (dotnetFlags != []) (builtins.toString dotnetFlags)} done - runHook postConfigure + echo "Writing lockfile..." + ${nuget-to-nix}/bin/nuget-to-nix "$HOME/nuget_pkgs" > "$deps_file" + echo "Succesfully wrote lockfile to: $deps_file" ''; - - buildPhase = args.buildPhase or '' - runHook preBuild - - for project in ''${projectFile[@]} ''${testProjectFile[@]}; do - dotnet build "$project" \ - -maxcpucount:${if enableParallelBuilding then "$NIX_BUILD_CORES" else "1"} \ - -p:BuildInParallel=${if enableParallelBuilding then "true" else "false"} \ - -p:ContinuousIntegrationBuild=true \ - -p:Deterministic=true \ - -p:Version=${args.version} \ - --configuration "$buildType" \ - --no-restore \ - "''${dotnetBuildFlags[@]}" \ - "''${dotnetFlags[@]}" - done - - runHook postBuild - ''; - - checkPhase = args.checkPhase or '' - runHook preCheck - - for project in ''${testProjectFile[@]}; do - ${lib.getBin dotnet-test-sdk}/bin/dotnet test "$project" \ - -maxcpucount:${if enableParallelBuilding then "$NIX_BUILD_CORES" else "1"} \ - -p:ContinuousIntegrationBuild=true \ - -p:Deterministic=true \ - --configuration "$buildType" \ - --no-build \ - --logger "console;verbosity=normal" \ - ${lib.optionalString (disabledTests != []) "--filter \"FullyQualifiedName!=${lib.concatStringsSep "&FullyQualifiedName!=" disabledTests}\""} \ - "''${dotnetTestFlags[@]}" \ - "''${dotnetFlags[@]}" - done - - runHook postCheck - ''; - - installPhase = args.installPhase or '' - runHook preInstall - - for project in ''${projectFile[@]}; do - dotnet publish "$project" \ - -p:ContinuousIntegrationBuild=true \ - -p:Deterministic=true \ - --output $out/lib/${args.pname} \ - --configuration "$buildType" \ - --no-build \ - --no-self-contained \ - "''${dotnetInstallFlags[@]}" \ - "''${dotnetFlags[@]}" - done - '' + lib.optionalString packNupkg '' - for project in ''${projectFile[@]}; do - dotnet pack "$project" \ - -p:ContinuousIntegrationBuild=true \ - -p:Deterministic=true \ - --output $out/share \ - --configuration "$buildType" \ - --no-build \ - "''${dotnetPackFlags[@]}" \ - "''${dotnetFlags[@]}" - done - '' + '' - runHook postInstall - ''; - - preFixup = '' - _wrapDotnetProgram() { - makeWrapper "$1" "$out/bin/$(basename "$executable")" \ - --set DOTNET_ROOT "${dotnet-runtime}" \ - --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath runtimeDeps}" \ - "''${gappsWrapperArgs[@]}" \ - "''${makeWrapperArgs[@]}" - } - '' + (if executables != null then '' - for executable in ''${executables[@]}; do - execPath="$out/lib/${args.pname}/$executable" - - if [[ -f "$execPath" && -x "$execPath" ]]; then - _wrapDotnetProgram $execPath - else - echo "Specified binary \"$executable\" is either not an executable, or does not exist!" - exit 1 - fi - done - '' else '' - for executable in $out/lib/${args.pname}/*; do - if [[ -f "$executable" && -x "$executable" && "$executable" != *"dll"* ]]; then - _wrapDotnetProgram $executable - fi - done - ''); - }); -in - package + } // args.passthru or {}; +}) diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/hooks/default.nix new file mode 100644 index 000000000000..76f7eea5bfec --- /dev/null +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/default.nix @@ -0,0 +1,62 @@ +{ lib +, callPackage +, makeSetupHook +, makeWrapper +, dotnet-sdk +, dotnet-test-sdk +, disabledTests +, nuget-source +, dotnet-runtime +, runtimeDeps +, buildType +}: + +{ + dotnetConfigureHook = callPackage ({ }: + makeSetupHook { + name = "dotnet-configure-hook"; + deps = [ dotnet-sdk nuget-source ]; + substitutions = { + nugetSource = nuget-source; + }; + } ./dotnet-configure-hook.sh) { }; + + dotnetBuildHook = callPackage ({ }: + makeSetupHook { + name = "dotnet-build-hook"; + deps = [ dotnet-sdk ]; + substitutions = { + inherit buildType; + }; + } ./dotnet-build-hook.sh) { }; + + dotnetCheckHook = callPackage ({ }: + makeSetupHook { + name = "dotnet-check-hook"; + deps = [ dotnet-test-sdk ]; + substitutions = { + inherit buildType; + disabledTests = lib.optionalString (disabledTests != []) + (lib.concatStringsSep "&FullyQualifiedName!=" disabledTests); + }; + } ./dotnet-check-hook.sh) { }; + + dotnetInstallHook = callPackage ({ }: + makeSetupHook { + name = "dotnet-install-hook"; + deps = [ dotnet-sdk ]; + substitutions = { + inherit buildType; + }; + } ./dotnet-install-hook.sh) { }; + + dotnetFixupHook = callPackage ({ }: + makeSetupHook { + name = "dotnet-fixup-hook"; + deps = [ dotnet-runtime makeWrapper ]; + substitutions = { + dotnetRuntime = dotnet-runtime; + runtimeDeps = lib.makeLibraryPath runtimeDeps; + }; + } ./dotnet-fixup-hook.sh) { }; +} diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh new file mode 100644 index 000000000000..a1dc80a77fd7 --- /dev/null +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh @@ -0,0 +1,41 @@ +declare -a projectFile testProjectFile dotnetBuildFlags dotnetFlags + +dotnetBuildHook() { + echo "Executing dotnetBuildHook" + + runHook preBuild + + if [ "${enableParallelBuilding-}" ]; then + maxCpuFlag="$NIX_BUILD_CORES" + parallelBuildFlag="true" + else + maxCpuFlag="1" + parallelBuildFlag="false" + fi + + if [ "${version-}" ]; then + versionFlag="-p:Version=${version-}" + fi + + for project in ${projectFile[@]} ${testProjectFile[@]}; do + env \ + dotnet build "$project" \ + -maxcpucount:$maxCpuFlag \ + -p:BuildInParallel=$parallelBuildFlag \ + -p:ContinuousIntegrationBuild=true \ + -p:Deterministic=true \ + --configuration "@buildType@" \ + --no-restore \ + ${versionFlag-} \ + "${dotnetBuildFlags[@]}" \ + "${dotnetFlags[@]}" + done + + runHook postBuild + + echo "Finished dotnetBuildHook" +} + +if [[ -z "${dontDotnetBuild-}" && -z "${buildPhase-}" ]]; then + buildPhase=dotnetBuildHook +fi diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-check-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-check-hook.sh new file mode 100644 index 000000000000..e3098908fe27 --- /dev/null +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-check-hook.sh @@ -0,0 +1,33 @@ +declare -a testProjectFile dotnetTestFlags dotnetFlags + +dotnetCheckHook() { + echo "Executing dotnetCheckHook" + + runHook preCheck + + if [ "${disabledTests-}" ]; then + disabledTestsFlag="--filter FullyQualifiedName!=@disabledTests@" + fi + + for project in ${testProjectFile[@]}; do + env \ + dotnet test "$project" \ + -maxcpucount:$maxCpuFlag \ + -p:ContinuousIntegrationBuild=true \ + -p:Deterministic=true \ + --configuration "@buildType@" \ + --no-build \ + --logger "console;verbosity=normal" \ + ${disabledTestsFlag-} \ + "${dotnetTestFlags[@]}" \ + "${dotnetFlags[@]}" + done + + runHook postCheck + + echo "Finished dotnetCheckHook" +} + +if [[ -z "${dontDotnetCheck-}" && -z "${checkPhase-}" ]]; then + checkPhase=dotnetCheckHook +fi diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh new file mode 100644 index 000000000000..3ca89fdc095e --- /dev/null +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh @@ -0,0 +1,32 @@ +declare -a projectFile testProjectFile dotnetRestoreFlags dotnetFlags + +dotnetConfigureHook() { + echo "Executing dotnetConfigureHook" + + runHook preConfigure + + if [ -z "${enableParallelBuilding-}" ]; then + parallelFlag="--disable-parallel" + fi + + export HOME=$(mktemp -d) + + for project in ${projectFile[@]} ${testProjectFile[@]}; do + env \ + dotnet restore "$project" \ + -p:ContinuousIntegrationBuild=true \ + -p:Deterministic=true \ + --source "@nugetSource@/lib" \ + ${parallelFlag-} \ + "${dotnetRestoreFlags[@]}" \ + "${dotnetFlags[@]}" + done + + runHook postConfigure + + echo "Finished dotnetConfigureHook" +} + +if [[ -z "${dontDotnetConfigure-}" && -z "${configurePhase-}" ]]; then + configurePhase=dotnetConfigureHook +fi diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh new file mode 100644 index 000000000000..f8bbb7b1805b --- /dev/null +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-fixup-hook.sh @@ -0,0 +1,42 @@ +declare -a makeWrapperArgs gappsWrapperArgs + +# First argument is the executable you want to wrap, +# the second is the destination for the wrapper. +wrapDotnetProgram() { + makeWrapper "$1" "$2" \ + --set "DOTNET_ROOT" "@dotnetRuntime@" \ + --suffix "LD_LIBRARY_PATH" : "@runtimeDeps@" \ + "${gappsWrapperArgs[@]}" \ + "${makeWrapperArgs[@]}" + + echo "Installed wrapper to: "$2"" +} + +dotnetFixupHook() { + echo "Executing dotnetFixupPhase" + + if [ "${executables-}" ]; then + for executable in ${executables[@]}; do + execPath="$out/lib/${pname-}/$executable" + + if [[ -f "$execPath" && -x "$execPath" ]]; then + wrapDotnetProgram "$execPath" "$out/bin/$(basename "$executable")" + else + echo "Specified binary \"$executable\" is either not an executable, or does not exist!" + exit 1 + fi + done + else + for executable in $out/lib/${pname-}/*; do + if [[ -f "$executable" && -x "$executable" && "$executable" != *"dll"* ]]; then + wrapDotnetProgram "$executable" "$out/bin/$(basename "$executable")" + fi + done + fi + + echo "Finished dotnetFixupPhase" +} + +if [[ -z "${dontDotnetFixup-}" ]]; then + preFixupPhases+=" dotnetFixupHook" +fi diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh new file mode 100644 index 000000000000..ed2c9160cd2c --- /dev/null +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-install-hook.sh @@ -0,0 +1,42 @@ +declare -a projectFile dotnetInstallFlags dotnetFlags + +dotnetInstallHook() { + echo "Executing dotnetInstallHook" + + runHook preInstall + + for project in ${projectFile[@]}; do + env \ + dotnet publish "$project" \ + -p:ContinuousIntegrationBuild=true \ + -p:Deterministic=true \ + --output "$out/lib/${pname}" \ + --configuration "@buildType@" \ + --no-build \ + --no-self-contained \ + "${dotnetInstallFlags[@]}" \ + "${dotnetFlags[@]}" + done + + if [[ "${packNupkg-}" ]]; then + for project in ${projectFile[@]}; do + env \ + dotnet pack "$project" \ + -p:ContinuousIntegrationBuild=true \ + -p:Deterministic=true \ + --output "$out/share" \ + --configuration "@buildType@" \ + --no-build \ + "${dotnetPackFlags[@]}" \ + "${dotnetFlags[@]}" + done + fi + + runHook postInstall + + echo "Finished dotnetInstallHook" +} + +if [[ -z "${dontDotnetInstall-}" && -z "${installPhase-}" ]]; then + installPhase=dotnetInstallHook +fi From 0b1856bfe3837b4509b180b4e9e51d19a88231ba Mon Sep 17 00:00:00 2001 From: IvarWithoutBones Date: Sat, 12 Feb 2022 01:19:20 +0100 Subject: [PATCH 2/2] buildDotnetModule: enable RestoreUseStaticGraphEvaluation This should speed up restore times a fair bit, especially for bigger projects. Roslyn also has it enabled by default already, so I don't expect any breakages from it. --- pkgs/build-support/dotnet/build-dotnet-module/default.nix | 1 + .../dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index e721b59decbe..d3561282d3fd 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -158,6 +158,7 @@ in stdenvNoCC.mkDerivation (args // { ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \ -p:ContinuousIntegrationBuild=true \ -p:Deterministic=true \ + -p:RestoreUseStaticGraphEvaluation=true \ --packages "$HOME/nuget_pkgs" \ ${lib.optionalString (dotnetRestoreFlags != []) (builtins.toString dotnetRestoreFlags)} \ ${lib.optionalString (dotnetFlags != []) (builtins.toString dotnetFlags)} diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh index 3ca89fdc095e..e0522dc95ce5 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-configure-hook.sh @@ -16,6 +16,7 @@ dotnetConfigureHook() { dotnet restore "$project" \ -p:ContinuousIntegrationBuild=true \ -p:Deterministic=true \ + -p:RestoreUseStaticGraphEvaluation=true \ --source "@nugetSource@/lib" \ ${parallelFlag-} \ "${dotnetRestoreFlags[@]}" \