From 038e86ffbb76bd54b922e68f5514102e36bec96b Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Sun, 2 Aug 2020 16:11:55 -0400 Subject: [PATCH 01/10] gcc: Fix building libgccjit on darwin, using strip -x The default `strip` invocation tries to strip global symbols from the library, and refuses because those are indirect symbol table references. --- pkgs/development/compilers/gcc/common/pre-configure.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/development/compilers/gcc/common/pre-configure.nix b/pkgs/development/compilers/gcc/common/pre-configure.nix index fff490148007..e4a7cfb4287c 100644 --- a/pkgs/development/compilers/gcc/common/pre-configure.nix +++ b/pkgs/development/compilers/gcc/common/pre-configure.nix @@ -49,3 +49,11 @@ lib.optionalString (hostPlatform.isSunOS && hostPlatform.is64bit) '' + lib.optionalString (hostPlatform.isDarwin) '' export ac_cv_func_aligned_alloc=no '' + +# In order to properly install on macOS Catalina, strip(1) upon +# installation must not remove external symbols, otherwise the install +# step errors with "symbols referenced by indirect symbol table +# entries that can't be stripped". ++ lib.optionalString (hostPlatform.isDarwin) '' + export STRIP='strip -x' +'' From 0d15ea9500664a3b4e98dc1c837a809b61785b77 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Sun, 2 Aug 2020 16:14:57 -0400 Subject: [PATCH 02/10] emacs: Use stdenv.libc instead of .glibc The latter doesn't exist on Darwin (and, presumably, other targets as well). This change allows emacsGcc from emacs-overlay to attempt to build where previously, the derivation would error out. --- pkgs/applications/editors/emacs/generic.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index 69bb51e10f20..0d7c31b8d86d 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -70,7 +70,7 @@ in stdenv.mkDerivation { libPath = lib.concatStringsSep ":" [ "${lib.getLib libgccjit}/lib/gcc/${targetPlatform.config}/${libgccjit.version}" "${lib.getLib stdenv.cc.cc}/lib" - "${lib.getLib stdenv.glibc}/lib" + "${lib.getLib stdenv.libc}/lib" ]; in '' substituteInPlace lisp/emacs-lisp/comp.el --replace \ From 175995986337e58e8ceec72d09515332fe363d3a Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Mon, 3 Aug 2020 23:38:57 -0400 Subject: [PATCH 03/10] gcc: On darwin, adjust IDs of installed .so files also Turns out that libgccjit gets installed as a .so file, which the gcc builder.sh didn't change: It only touched .dylib files; that means that anything linking in libgccjit.so would receive an "Image not found" error at load time. With this change, we invoke `install_name_tool` on .so files too, adjusting their dynamic linker ID, so that they too can be found. --- pkgs/development/compilers/gcc/builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/compilers/gcc/builder.sh b/pkgs/development/compilers/gcc/builder.sh index 455870208448..80e07c7f93f6 100644 --- a/pkgs/development/compilers/gcc/builder.sh +++ b/pkgs/development/compilers/gcc/builder.sh @@ -251,7 +251,7 @@ postInstall() { fi if type "install_name_tool"; then - for i in "${!outputLib}"/lib/*.*.dylib; do + for i in "${!outputLib}"/lib/*.*.dylib "${!outputLib}"/lib/*.so.[0-9]; do install_name_tool -id "$i" "$i" || true for old_path in $(otool -L "$i" | grep "$out" | awk '{print $1}'); do new_path=`echo "$old_path" | sed "s,$out,${!outputLib},"` From 861f27018d9d6d131f4a717776e81dcebc38b9f8 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Wed, 26 Aug 2020 20:40:06 -0400 Subject: [PATCH 04/10] emacs: Fix paths to bintools on Darwin This changes PATH such that the correct linker can be found to construct .eln files at runtime. --- pkgs/applications/editors/emacs/generic.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index 0d7c31b8d86d..8e2c34988b9b 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -11,7 +11,7 @@ , libtiff, librsvg, gconf, libxml2, imagemagick, gnutls, libselinux , alsaLib, cairo, acl, gpm, AppKit, GSS, ImageIO, m17n_lib, libotf , jansson, harfbuzz -, libgccjit, targetPlatform, binutils, binutils-unwrapped, makeWrapper # native-comp params +, libgccjit, targetPlatform, binutils, clang ? null, binutils-unwrapped, makeWrapper # native-comp params , systemd ? null , withX ? !stdenv.isDarwin , withNS ? stdenv.isDarwin @@ -158,7 +158,7 @@ in stdenv.mkDerivation { '') (lib.optionalString nativeComp '' - wrapProgram $out/bin/emacs-* --prefix PATH : "${lib.makeBinPath [ binutils binutils-unwrapped ]}" + wrapProgram $out/bin/emacs-* --prefix PATH : "${lib.makeBinPath [ clang.bintools binutils binutils-unwrapped ]}" '') ]; From a891ae41b36604e20c9ec130940728d673f5ad30 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Wed, 26 Aug 2020 20:37:35 -0400 Subject: [PATCH 05/10] emacs: Set native-comp library path as linker flags instead Since Darwin's linker does not understand LIBRARY_PATH, we have to set the library path as explicit linker flags: This requires a very recent feature/native-comp emacs revision, but it runs on Darwin and correctly compiles files at runtime. --- pkgs/applications/editors/emacs/generic.nix | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index 8e2c34988b9b..dcbefb884317 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -67,18 +67,18 @@ in stdenv.mkDerivation { # Make native compilation work both inside and outside of nix build (lib.optionalString nativeComp (let - libPath = lib.concatStringsSep ":" [ - "${lib.getLib libgccjit}/lib/gcc/${targetPlatform.config}/${libgccjit.version}" - "${lib.getLib stdenv.cc.cc}/lib" - "${lib.getLib stdenv.libc}/lib" - ]; + libPath = (lib.concatStringsSep " " + (builtins.map (x: ''\"-L${x}\"'') [ + "${lib.getLib libgccjit}/lib" + "${lib.getLib libgccjit}/lib/gcc/${targetPlatform.config}/${libgccjit.version}" + "${lib.getLib stdenv.cc.cc}/lib" + "${lib.getLib stdenv.libc}/lib" + ])); in '' substituteInPlace lisp/emacs-lisp/comp.el --replace \ - "(defcustom comp-async-env-modifier-form nil" \ - "(defcustom comp-async-env-modifier-form '((setenv \"LIBRARY_PATH\" (string-join (seq-filter (lambda (v) (null (eq v nil))) (list (getenv \"LIBRARY_PATH\") \"${libPath}\")) \":\")))" - + "(defcustom comp-native-driver-options nil" \ + "(defcustom comp-native-driver-options '(${libPath})" '')) - "" ]; From 41e34e76d8802feaa9c13f88a5f032c529fc74e9 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Thu, 27 Aug 2020 12:38:31 -0400 Subject: [PATCH 06/10] gcc: Only use strip -x if building libgccjit on darwin It's not necessary to use strip -x otherwise, so let's just use it for the JIT library. --- pkgs/development/compilers/gcc/9/default.nix | 2 +- .../compilers/gcc/common/pre-configure.nix | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkgs/development/compilers/gcc/9/default.nix b/pkgs/development/compilers/gcc/9/default.nix index 5f0a69583a18..20c34c1f4506 100644 --- a/pkgs/development/compilers/gcc/9/default.nix +++ b/pkgs/development/compilers/gcc/9/default.nix @@ -181,7 +181,7 @@ stdenv.mkDerivation ({ preConfigure = import ../common/pre-configure.nix { inherit (stdenv) lib; - inherit version hostPlatform gnatboot langAda langGo; + inherit version hostPlatform gnatboot langAda langGo langJit; }; dontDisableStatic = true; diff --git a/pkgs/development/compilers/gcc/common/pre-configure.nix b/pkgs/development/compilers/gcc/common/pre-configure.nix index e4a7cfb4287c..1c65b4a8ba64 100644 --- a/pkgs/development/compilers/gcc/common/pre-configure.nix +++ b/pkgs/development/compilers/gcc/common/pre-configure.nix @@ -2,6 +2,7 @@ , gnatboot ? null , langAda ? false , langJava ? false +, langJit ? false , langGo }: assert langJava -> lib.versionOlder version "7"; @@ -50,10 +51,10 @@ lib.optionalString (hostPlatform.isSunOS && hostPlatform.is64bit) '' export ac_cv_func_aligned_alloc=no '' -# In order to properly install on macOS Catalina, strip(1) upon -# installation must not remove external symbols, otherwise the install -# step errors with "symbols referenced by indirect symbol table -# entries that can't be stripped". -+ lib.optionalString (hostPlatform.isDarwin) '' +# In order to properly install libgccjit on macOS Catalina, strip(1) +# upon installation must not remove external symbols, otherwise the +# install step errors with "symbols referenced by indirect symbol +# table entries that can't be stripped". ++ lib.optionalString (hostPlatform.isDarwin && langJit) '' export STRIP='strip -x' '' From ad7a3fb3f691a12fc95ecee91995fd2f440701b1 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Sat, 29 Aug 2020 12:22:14 -0400 Subject: [PATCH 07/10] emacs: Use stdenv.cc.libc instead of plain .libc This is the portable way to address the system-appropriate libc, better than conditionalizing by (darwin/GNU) system or using either one. --- pkgs/applications/editors/emacs/generic.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index dcbefb884317..be4b343f4e58 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -72,7 +72,7 @@ in stdenv.mkDerivation { "${lib.getLib libgccjit}/lib" "${lib.getLib libgccjit}/lib/gcc/${targetPlatform.config}/${libgccjit.version}" "${lib.getLib stdenv.cc.cc}/lib" - "${lib.getLib stdenv.libc}/lib" + "${lib.getLib stdenv.cc.libc}/lib" ])); in '' substituteInPlace lisp/emacs-lisp/comp.el --replace \ From 3384837123766c806c0b925053799e641077f570 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Sun, 30 Aug 2020 10:58:50 -0400 Subject: [PATCH 08/10] emacs: Use stdenv's bintools instead of clang This way, we don't have to drag clang or binutils/binutils-wrapped into the emacs closure, and can instead rely on using the correct one for the platform we're running on. Co-authored-by: Matthew Bauer --- pkgs/applications/editors/emacs/generic.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index be4b343f4e58..46c6a982d404 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -11,7 +11,7 @@ , libtiff, librsvg, gconf, libxml2, imagemagick, gnutls, libselinux , alsaLib, cairo, acl, gpm, AppKit, GSS, ImageIO, m17n_lib, libotf , jansson, harfbuzz -, libgccjit, targetPlatform, binutils, clang ? null, binutils-unwrapped, makeWrapper # native-comp params +, libgccjit, targetPlatform, makeWrapper # native-comp params , systemd ? null , withX ? !stdenv.isDarwin , withNS ? stdenv.isDarwin @@ -158,7 +158,7 @@ in stdenv.mkDerivation { '') (lib.optionalString nativeComp '' - wrapProgram $out/bin/emacs-* --prefix PATH : "${lib.makeBinPath [ clang.bintools binutils binutils-unwrapped ]}" + wrapProgram $out/bin/emacs-* --prefix PATH : "${lib.makeBinPath [ stdenv.cc.bintools stdenv.cc.bintools.bintools ]}" '') ]; From 6ad323d0a02cf2afd857a4f30cfce8277a0e11cd Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Mon, 31 Aug 2020 00:47:33 -0400 Subject: [PATCH 09/10] emacs: use -B flags for native compilation dependencies The -B flag to gcc (and libgccjit) allows us to specify where it can find things it needs to correctly compile code (both programs and libraries) without adjusting any environmental flags: So, no need to wrap the program for a PATH entry containing binutils, and no need to explicitly pass a linker path anymore. --- pkgs/applications/editors/emacs/generic.nix | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index 46c6a982d404..b2d9b9a1367f 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -68,11 +68,15 @@ in stdenv.mkDerivation { # Make native compilation work both inside and outside of nix build (lib.optionalString nativeComp (let libPath = (lib.concatStringsSep " " - (builtins.map (x: ''\"-L${x}\"'') [ + (builtins.map (x: ''\"-B${x}\"'') [ "${lib.getLib libgccjit}/lib" "${lib.getLib libgccjit}/lib/gcc/${targetPlatform.config}/${libgccjit.version}" "${lib.getLib stdenv.cc.cc}/lib" "${lib.getLib stdenv.cc.libc}/lib" + "${lib.getBin stdenv.cc.cc}" + "${lib.getBin stdenv.cc.cc}" + "${lib.getBin stdenv.cc.bintools}" + "${lib.getBin stdenv.cc.bintools.bintools}" ])); in '' substituteInPlace lisp/emacs-lisp/comp.el --replace \ @@ -156,11 +160,6 @@ in stdenv.mkDerivation { "$out/bin/emacs" patchelf --add-needed "libXcursor.so.1" "$out/bin/emacs" '') - - (lib.optionalString nativeComp '' - wrapProgram $out/bin/emacs-* --prefix PATH : "${lib.makeBinPath [ stdenv.cc.bintools stdenv.cc.bintools.bintools ]}" - '') - ]; passthru = { From 04fffd6cae88e676a13fa94e1f7abdaa3156e707 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Mon, 31 Aug 2020 00:47:33 -0400 Subject: [PATCH 10/10] emacs: use -B flags for native compilation dependencies The -B flag to gcc (and libgccjit) allows us to specify where it can find things it needs to correctly compile code (both programs and libraries) without adjusting any environmental flags: So, no need to wrap the program for a PATH entry containing binutils, and no need to explicitly pass a linker path anymore. --- pkgs/applications/editors/emacs/generic.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index b2d9b9a1367f..9a19fb787e12 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -67,13 +67,14 @@ in stdenv.mkDerivation { # Make native compilation work both inside and outside of nix build (lib.optionalString nativeComp (let - libPath = (lib.concatStringsSep " " + backendPath = (lib.concatStringsSep " " (builtins.map (x: ''\"-B${x}\"'') [ + # Paths necessary so the JIT compiler finds its libraries: "${lib.getLib libgccjit}/lib" - "${lib.getLib libgccjit}/lib/gcc/${targetPlatform.config}/${libgccjit.version}" - "${lib.getLib stdenv.cc.cc}/lib" + "${lib.getLib libgccjit}/lib/gcc" "${lib.getLib stdenv.cc.libc}/lib" - "${lib.getBin stdenv.cc.cc}" + + # Executable paths necessary for compilation (ld, as): "${lib.getBin stdenv.cc.cc}" "${lib.getBin stdenv.cc.bintools}" "${lib.getBin stdenv.cc.bintools.bintools}" @@ -81,7 +82,7 @@ in stdenv.mkDerivation { in '' substituteInPlace lisp/emacs-lisp/comp.el --replace \ "(defcustom comp-native-driver-options nil" \ - "(defcustom comp-native-driver-options '(${libPath})" + "(defcustom comp-native-driver-options '(${backendPath})" '')) "" ];