Merge pull request #107977 from corngood/msbuild

pkgs/development/compilers/dotnet/default.nix

@@ -124,11 +124,11 @@ rec {
   };
 
   sdk_5_0 = buildNetCoreSdk {
-    version = "5.0.100";
+    version = "5.0.200";
     sha512 = {
-      x86_64-linux = "bec37bfb327c45cc01fd843ef93b22b556f753b04724bba501622df124e7e144c303a4d7e931b5dbadbd4f7b39e5adb8f601cb6293e317ad46d8fe7d52aa9a09";
-      aarch64-linux = "5fceac0a9468097d66af25516da597eb4836b294ed1647ba272ade5c8faea2ed977a95d9ce720c44d71607fa3a0cf9de55afe0e66c0c89ab1cc6736945978204";
-      x86_64-darwin = "69ccc7c686ac06f6c658d118f59cf1a0e7284b4570375dd88d3e3043098e311745922301f2650d159624d09c4d39a1f3cbdd5daee0e408eef915de839e3bce8f";
+      x86_64-linux = "0g7zcmkcdwc11h42m6hq8d0w55nnvnsmj3dc16829q55cp7l7kggmjljnd9slx7r7nrsyi7yy8brwh8n4kfi5998pdyb09fzhq5w60d";
+      aarch64-linux = "2zy6nxiw313g2sbmnkg76r64llbk2w2wcsa6frq535zbviw52zf163jvw2687rpiw4szdizf337l3b0qa0396abw5dhq2czqlxjyjv8";
+      x86_64-darwin = "2p0yxplafhi5ks38pq8nyi43kpv4l4npa718rvcvl57qs76j0dqlk1s4wdw7msx8g7xxy1aal47zy9rxvlypmgwx4dnp339cmbd6mf6";
     };
   };
 }

pkgs/development/compilers/roslyn/create-deps.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p dotnet-sdk_5 -p jq -p xmlstarlet -p curl
+set -euo pipefail
+
+cat << EOL
+{ fetchurl }: [
+EOL
+
+tmpdir="$(mktemp -d -p "$(pwd)")" # must be under source root
+trap 'rm -rf "$tmpdir"' EXIT
+
+HOME="$tmpdir" dotnet msbuild -t:restore -p:Configuration=Release -p:RestorePackagesPath="$tmpdir"/.nuget/packages \
+        -p:RestoreNoCache=true -p:RestoreForce=true \
+        src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj >&2
+
+mapfile -t repos < <(
+    xmlstarlet sel -t -v 'configuration/packageSources/add/@value' -n NuGet.config "$tmpdir"/.nuget/NuGet/NuGet.Config |
+        while IFS= read index
+        do
+            curl --compressed -fsL "$index" | \
+                jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"'
+        done
+)
+
+cd "$tmpdir/.nuget/packages"
+for package in *
+do
+    cd "$package"
+    for version in *
+    do
+        found=false
+        for repo in "${repos[@]}"
+        do
+            url="$repo$package/$version/$package.$version.nupkg"
+            if curl -fsL "$url" -o /dev/null
+            then
+                found=true
+                break
+            fi
+        done
+
+        if ! $found
+        then
+            echo "couldn't find $package $version" >&2
+            exit 1
+        fi
+
+        sha256=$(nix-prefetch-url "$url" 2>/dev/null)
+        cat << EOL
+  {
+    name = "$package";
+    version = "$version";
+    src = fetchurl {
+      url = "$url";
+      sha256 = "$sha256";
+    };
+  }
+EOL
+    done
+    cd ..
+done
+
+cat << EOL
+]
+EOL

pkgs/development/compilers/roslyn/default.nix

@@ -0,0 +1,121 @@
+{ lib, stdenv
+, fetchFromGitHub
+, fetchurl
+, mono
+, dotnet-sdk_5
+, makeWrapper
+, dotnetPackages
+, unzip
+, writeText
+, symlinkJoin
+}:
+
+let
+
+  deps = map (package: stdenv.mkDerivation (with package; {
+    pname = name;
+    inherit version src;
+
+    buildInputs = [ unzip ];
+    unpackPhase = ''
+      unzip -o $src
+      chmod -R u+r .
+      function traverseRename () {
+        for e in *
+        do
+          t="$(echo "$e" | sed -e "s/%20/\ /g" -e "s/%2B/+/g")"
+          [ "$t" != "$e" ] && mv -vn "$e" "$t"
+          if [ -d "$t" ]
+          then
+            cd "$t"
+            traverseRename
+            cd ..
+          fi
+        done
+      }
+
+      traverseRename
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      package=$out/lib/dotnet/${name}/${version}
+      mkdir -p $package
+      cp -r . $package
+      echo "{}" > $package/.nupkg.metadata
+
+      runHook postInstall
+    '';
+
+    dontFixup = true;
+  }))
+    (import ./deps.nix { inherit fetchurl; });
+
+  nuget-config = writeText "NuGet.Config" ''
+    <?xml version="1.0" encoding="utf-8"?>
+    <configuration>
+      <packageSources>
+        <clear />
+      </packageSources>
+    </configuration>
+  '';
+
+  packages = symlinkJoin { name = "roslyn-deps"; paths = deps; };
+
+  packageVersion = "3.10.0";
+
+in stdenv.mkDerivation rec {
+
+  pname = "roslyn";
+  version = "${packageVersion}-1.21102.26";
+
+  src = fetchFromGitHub {
+    owner = "dotnet";
+    repo = "roslyn";
+    rev = "v${version}";
+    sha256 = "0yf4f4vpqn9lixr37lkp29m2mk51xcm3ysv2ag332xn6zm5zpm2b";
+  };
+
+  nativeBuildInputs = [ makeWrapper dotnet-sdk_5 unzip ];
+
+  buildPhase = ''
+    runHook preBuild
+
+    rm NuGet.config
+    install -m644 -D ${nuget-config} fake-home/.nuget/NuGet/NuGet.Config
+    ln -s ${packages}/lib/dotnet fake-home/.nuget/packages
+    HOME=$(pwd)/fake-home dotnet add \
+      src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj \
+      package -n -v 5.10.0-preview.2.7169 nuget.build.tasks.pack
+    HOME=$(pwd)/fake-home dotnet msbuild -r -v:m -t:pack \
+      -p:Configuration=Release \
+      -p:RepositoryUrl="${meta.homepage}" \
+      -p:RepositoryCommit="v${version}" \
+      src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    pkg=$out/lib/dotnet/microsoft.net.compilers.toolset/${packageVersion}
+    mkdir -p $out/bin $pkg
+    unzip -q artifacts/packages/Release/Shipping/Microsoft.Net.Compilers.Toolset.${packageVersion}-dev.nupkg \
+      -d $pkg
+    # nupkg has 0 permissions for a bunch of things
+    chmod -R +rw $pkg
+
+    makeWrapper ${mono}/bin/mono $out/bin/csc \
+      --add-flags "$pkg/tasks/net472/csc.exe"
+    makeWrapper ${mono}/bin/mono $out/bin/vbs \
+      --add-flags "$pkg/tasks/net472/vbs.exe"
+  '';
+
+  meta = with lib; {
+    description = ".NET C# and Visual Basic compiler";
+    homepage = "https://github.com/dotnet/roslyn";
+    platforms = platforms.linux;
+    license = licenses.mit;
+    maintainers = with maintainers; [ corngood ];
+  };
+}

pkgs/development/tools/build-managers/msbuild/create-deps.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jq -p xmlstarlet -p curl
+set -euo pipefail
+
+cat << EOL
+{ fetchurl }: [
+EOL
+
+mapfile -t repos < <(
+    xmlstarlet sel -t -v 'configuration/packageSources/add/@value' -n NuGet.config |
+        while IFS= read index
+        do
+            curl --compressed -fsL "$index" | \
+                jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"'
+        done
+    )
+
+find .packages fake-home/.nuget/packages -name \*.nupkg -printf '%P\n' | sort -u |
+    while IFS= read file
+    do
+        packagedir=$(dirname $file)
+        version=$(basename $packagedir)
+        package=$(dirname $packagedir)
+
+        found=false
+        for repo in "${repos[@]}"
+        do
+            url="$repo$package/$version/$package.$version.nupkg"
+            if curl -fsL "$url" -o /dev/null
+            then
+                found=true
+                break
+            fi
+        done
+
+        if ! $found
+        then
+            echo "couldn't find $package $version" >&2
+            exit 1
+        fi
+
+        sha256=$(nix-prefetch-url "$url" 2>/dev/null)
+        cat << EOL
+  {
+    name = "$package";
+    version = "$version";
+    src = fetchurl {
+      url = "$url";
+      sha256 = "$sha256";
+    };
+  }
+EOL
+    done
+
+cat << EOL
+]
+EOL

pkgs/development/tools/build-managers/msbuild/default.nix

@@ -1,23 +1,33 @@
-{ lib, stdenv, fetchurl, makeWrapper, glibcLocales, mono, dotnetPackages, unzip, dotnet-sdk }:
+{ lib, stdenv, fetchurl, fetchpatch, makeWrapper, glibcLocales, mono, dotnetPackages, unzip, dotnet-sdk, writeText, roslyn }:
 
 let
 
   xplat = fetchurl {
-    url = "https://github.com/mono/msbuild/releases/download/0.07/mono_msbuild_xplat-master-8f608e49.zip";
-    sha256 = "1jxq3fk9a6q2a8i9zacxaz3fkvc22i9qvzlpa7wbb95h42g0ffhq";
+    url = "https://github.com/mono/msbuild/releases/download/0.08/mono_msbuild_6.4.0.208.zip";
+    sha256 = "05k7qmnhfvrdgyjn6vp81jb97y21m261jnwdyqpjqpcmzz18j93g";
   };
 
-  deps = import ./nuget.nix { inherit fetchurl; };
+  deps = map (package: package.src)
+    (import ./deps.nix { inherit fetchurl; });
+
+  nuget-config = writeText "NuGet.config" ''
+    <?xml version="1.0" encoding="utf-8"?>
+    <configuration>
+      <packageSources>
+        <clear />
+      </packageSources>
+    </configuration>
+  '';
 
 in
 
 stdenv.mkDerivation rec {
   pname = "msbuild";
-  version = "16.3+xamarinxplat.2019.07.26.14.57";
+  version = "16.8+xamarinxplat.2020.07.30.15.02";
 
   src = fetchurl {
     url = "https://download.mono-project.com/sources/msbuild/msbuild-${version}.tar.xz";
-    sha256 = "1zcdfx4xsh62wj3g1jc2an0lppsfs691lz4dv05xbgi01aq1hk6a";
+    sha256 = "10amyca78b6pjfsy54b1rgwz2c1bx0sfky9zhldvzy4divckp25g";
   };
 
   nativeBuildInputs = [
@@ -32,25 +42,34 @@ stdenv.mkDerivation rec {
     makeWrapper
   ];
 
-  # https://aur.archlinux.org/cgit/aur.git/tree/PKGBUILD?h=msbuild
-  phases = ["unpackPhase" "buildPhase" "installPhase" "installCheckPhase"];
-
   # https://github.com/NixOS/nixpkgs/issues/38991
   # bash: warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8)
   LOCALE_ARCHIVE = lib.optionalString stdenv.isLinux
       "${glibcLocales}/lib/locale/locale-archive";
 
+  patches = [
+    (fetchpatch {
+      url = "https://github.com/mono/msbuild/commit/cad85cefabdaa001fb4bdbea2f5bf1d1cdb83c9b.patch";
+      sha256 = "1s8agc7nxxs69b3fl1v1air0c4dpig3hy4sk11l1560jrlx06dhh";
+    })
+  ];
+
+  postPatch = ''
+    sed -i -e "/<\/projectImportSearchPaths>/a <property name=\"MSBuildExtensionsPath\" value=\"$out/lib/mono/xbuild\"/>" \
+      src/MSBuild/app.config
+  '';
+
   buildPhase = ''
     # nuget would otherwise try to base itself in /homeless-shelter
     export HOME=$(pwd)/fake-home
 
+    cp ${nuget-config} NuGet.config
+    nuget sources Add -Name nixos -Source $(pwd)/nixos
+
     for package in ${toString deps}; do
       nuget add $package -Source nixos
     done
 
-    nuget sources Disable -Name "nuget.org"
-    nuget sources Add -Name nixos -Source $(pwd)/nixos
-
     # license check is case sensitive
     mv LICENSE license.bak && mv license.bak license
 
@@ -64,27 +83,25 @@ stdenv.mkDerivation rec {
     # overwrite the file
     echo "#!${stdenv.shell}" > eng/common/dotnet-install.sh
 
-    # msbuild response files to use only the nixos source
-    echo "/p:RestoreSources=nixos" > artifacts/mono-msbuild/MSBuild.rsp
-    echo "/p:RestoreSources=nixos" > src/MSBuild/MSBuild.rsp
-
     # not patchShebangs, there is /bin/bash in the body of the script as well
     substituteInPlace ./eng/cibuild_bootstrapped_msbuild.sh --replace /bin/bash ${stdenv.shell}
 
     # DisableNerdbankVersioning https://gitter.im/Microsoft/msbuild/archives/2018/06/27?at=5b33dbc4ce3b0f268d489bfa
     # TODO there are some (many?) failing tests
     ./eng/cibuild_bootstrapped_msbuild.sh --host_type mono --configuration Release --skip_tests /p:DisableNerdbankVersioning=true
+    patchShebangs stage1/mono-msbuild/msbuild
   '';
 
   installPhase = ''
-    mono artifacts/mono-msbuild/MSBuild.dll mono/build/install.proj /p:MonoInstallPrefix="$out" /p:Configuration=Release-MONO
+    stage1/mono-msbuild/msbuild mono/build/install.proj /p:MonoInstallPrefix="$out" /p:Configuration=Release-MONO
 
-    ln -s ${mono}/lib/mono/msbuild/Current/bin/Roslyn $out/lib/mono/msbuild/Current/bin/Roslyn
+    ln -s ${roslyn}/lib/dotnet/microsoft.net.compilers.toolset/*/tasks/net472 $out/lib/mono/msbuild/Current/bin/Roslyn
 
     makeWrapper ${mono}/bin/mono $out/bin/msbuild \
-      --set MSBuildExtensionsPath $out/lib/mono/xbuild \
       --set-default MONO_GC_PARAMS "nursery-size=64m" \
       --add-flags "$out/lib/mono/msbuild/15.0/bin/MSBuild.dll"
+
+    ln -s $(find ${dotnet-sdk} -name libhostfxr.so) $out/lib/mono/msbuild/Current/bin/SdkResolvers/Microsoft.DotNet.MSBuildSdkResolver/
   '';
 
   doInstallCheck = true;
@@ -130,4 +147,3 @@ EOF
     platforms = platforms.unix;
   };
 }
-

pkgs/top-level/all-packages.nix

@@ -11050,6 +11050,8 @@ in
 
   monoDLLFixer = callPackage ../build-support/mono-dll-fixer { };
 
+  roslyn = callPackage ../development/compilers/roslyn { mono = mono6; };
+
   msbuild = callPackage ../development/tools/build-managers/msbuild { mono = mono6; };
 
   mosml = callPackage ../development/compilers/mosml { };