From 2c149715fe2cb456c03d6495c474e8db306682cb Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Mon, 12 Jun 2023 01:12:17 -0400 Subject: [PATCH 1/7] darwin.xnu: provide additional headers needed by Libsystem --- pkgs/os-specific/darwin/apple-source-releases/xnu/default.nix | 3 +++ .../darwin/apple-source-releases/xnu/headers-arm64.txt | 2 ++ .../darwin/apple-source-releases/xnu/headers-x86_64.txt | 2 ++ 3 files changed, 7 insertions(+) diff --git a/pkgs/os-specific/darwin/apple-source-releases/xnu/default.nix b/pkgs/os-specific/darwin/apple-source-releases/xnu/default.nix index 62003a6e4a6c..b05d973bb3d3 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/xnu/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/xnu/default.nix @@ -116,6 +116,9 @@ appleDerivation' (if headersOnly then stdenvNoCC else stdenv) ( cp EXTERNAL_HEADERS/Availability*.h $out/System/Library/Frameworks/Kernel.framework/Versions/A/Headers/ cp -r EXTERNAL_HEADERS/corecrypto $out/include + # These headers are needed by Libsystem. + cp libsyscall/wrappers/{spawn/spawn.h,libproc/libproc.h} $out/include + # Build the mach headers we crave export SRCROOT=$PWD/libsyscall export DERIVED_SOURCES_DIR=$out/include diff --git a/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-arm64.txt b/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-arm64.txt index fe17d8784e41..23ae12a9b057 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-arm64.txt +++ b/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-arm64.txt @@ -637,6 +637,7 @@ libkern/os/reason_private.h libkern/os/refcnt.h libkern/os/refcnt_internal.h libkern/os/trace.h +libproc.h mach/arm/_structs.h mach/arm/asm.h mach/arm/boolean.h @@ -1270,6 +1271,7 @@ servers/ls_defs.h servers/netname.h servers/netname_defs.h servers/nm_defs.h +spawn.h sys/_endian.h sys/_posix_availability.h sys/_select.h diff --git a/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-x86_64.txt b/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-x86_64.txt index 93c0dbb18bf7..8d7d9be7f218 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-x86_64.txt +++ b/pkgs/os-specific/darwin/apple-source-releases/xnu/headers-x86_64.txt @@ -554,6 +554,7 @@ libkern/os/object.h libkern/os/object_private.h libkern/os/overflow.h libkern/os/trace.h +libproc.h mach/audit_triggers.defs mach/boolean.h mach/bootstrap.h @@ -1116,6 +1117,7 @@ servers/ls_defs.h servers/netname.h servers/netname_defs.h servers/nm_defs.h +spawn.h sys/_endian.h sys/_posix_availability.h sys/_select.h From a48c2d3e9225d779ae2ba46bb73c1f81ca9246e8 Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Sun, 11 Jun 2023 23:11:14 -0400 Subject: [PATCH 2/7] darwin.libmalloc: add at 116.50.8 --- .../darwin/apple-source-releases/default.nix | 2 ++ .../darwin/apple-source-releases/libmalloc/default.nix | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 pkgs/os-specific/darwin/apple-source-releases/libmalloc/default.nix diff --git a/pkgs/os-specific/darwin/apple-source-releases/default.nix b/pkgs/os-specific/darwin/apple-source-releases/default.nix index 40f3fd2478a5..1f293444a696 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/default.nix @@ -19,6 +19,7 @@ let Libinfo = "503.50.4"; Libsystem = "1238.60.2"; removefile = "45"; + libmalloc = "116.50.8"; libresolv = "64"; libplatform = "126.50.8"; mDNSResponder = "765.50.9"; @@ -273,6 +274,7 @@ developerToolsPackages_11_3_1 // macosPackages_11_0_1 // { Libinfo = applePackage "Libinfo" "osx-10.11.6" "sha256-6F7wiwerv4nz/xXHtp1qCHSaFzZgzcRN+jbmXA5oWOQ=" {}; Libm = applePackage "Libm" "osx-10.7.4" "sha256-KjMETfT4qJm0m0Ux/F6Rq8bI4Q4UVnFx6IKbKxXd+Es=" {}; Libnotify = applePackage "Libnotify" "osx-10.12.6" "sha256-6wvMBxAUfiYcQtmlfYCj1d3kFmFM/jdboTd7hRvi3e4=" {}; + libmalloc = applePackage "libmalloc" "osx-10.12.6" "sha256-brfG4GEF2yZipKdhlPq6DhT2z5hKYSb2MAmffaikdO4=" {}; libplatform = applePackage "libplatform" "osx-10.12.6" "sha256-6McMTjw55xtnCsFI3AB1osRagnuB5pSTqeMKD3gpGtM=" {}; libpthread = applePackage "libpthread" "osx-10.12.6" "sha256-QvJ9PERmrCWBiDmOWrLvQUKZ4JxHuh8gS5nlZKDLqE8=" {}; libresolv = applePackage "libresolv" "osx-10.12.6" "sha256-FtvwjJKSFX6j9APYPC8WLXVOjbHLZa1Gcoc8yxLy8qE=" {}; diff --git a/pkgs/os-specific/darwin/apple-source-releases/libmalloc/default.nix b/pkgs/os-specific/darwin/apple-source-releases/libmalloc/default.nix new file mode 100644 index 000000000000..8b362a2edd54 --- /dev/null +++ b/pkgs/os-specific/darwin/apple-source-releases/libmalloc/default.nix @@ -0,0 +1,10 @@ +{ appleDerivation', stdenvNoCC }: + +# Unfortunately, buiding libmalloc is not feasible due to its use of non-public headers, but its +# headers are needed by Libsystem. +appleDerivation' stdenvNoCC { + installPhase = '' + mkdir -p $out/include + cp -R include/malloc $out/include/ + ''; +} From 81324e2ec3fba15669f8decb18257a9360bd44c7 Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Sun, 11 Jun 2023 23:45:24 -0400 Subject: [PATCH 3/7] darwin.libmalloc: add at 317.40.8 for the 11.0 SDK --- pkgs/os-specific/darwin/apple-source-releases/default.nix | 4 +++- .../os-specific/darwin/apple-source-releases/macos-11.0.1.nix | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/os-specific/darwin/apple-source-releases/default.nix b/pkgs/os-specific/darwin/apple-source-releases/default.nix index 1f293444a696..a277ebfd580f 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/default.nix @@ -274,7 +274,9 @@ developerToolsPackages_11_3_1 // macosPackages_11_0_1 // { Libinfo = applePackage "Libinfo" "osx-10.11.6" "sha256-6F7wiwerv4nz/xXHtp1qCHSaFzZgzcRN+jbmXA5oWOQ=" {}; Libm = applePackage "Libm" "osx-10.7.4" "sha256-KjMETfT4qJm0m0Ux/F6Rq8bI4Q4UVnFx6IKbKxXd+Es=" {}; Libnotify = applePackage "Libnotify" "osx-10.12.6" "sha256-6wvMBxAUfiYcQtmlfYCj1d3kFmFM/jdboTd7hRvi3e4=" {}; - libmalloc = applePackage "libmalloc" "osx-10.12.6" "sha256-brfG4GEF2yZipKdhlPq6DhT2z5hKYSb2MAmffaikdO4=" {}; + libmalloc = if stdenv.isx86_64 then + applePackage "libmalloc" "osx-10.12.6" "sha256-brfG4GEF2yZipKdhlPq6DhT2z5hKYSb2MAmffaikdO4=" {} + else macosPackages_11_0_1.libmalloc; libplatform = applePackage "libplatform" "osx-10.12.6" "sha256-6McMTjw55xtnCsFI3AB1osRagnuB5pSTqeMKD3gpGtM=" {}; libpthread = applePackage "libpthread" "osx-10.12.6" "sha256-QvJ9PERmrCWBiDmOWrLvQUKZ4JxHuh8gS5nlZKDLqE8=" {}; libresolv = applePackage "libresolv" "osx-10.12.6" "sha256-FtvwjJKSFX6j9APYPC8WLXVOjbHLZa1Gcoc8yxLy8qE=" {}; diff --git a/pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix b/pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix index 1637ed783895..0a70e648695d 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix @@ -28,6 +28,7 @@ hfs = applePackage' "hfs" "556.41.1" "macos-11.0.1" "0a0s6b12b0q07wslfifna0bj51d libclosure = applePackage' "libclosure" "78" "macos-11.0.1" "0vf9n0k3m8dbprv1bf45zqg0g43bidy2i5z1v9a826bsf8lv7am7" {}; libdispatch = applePackage' "libdispatch" "1271.40.12" "macos-11.0.1" "1ck5srcjapg18vqb8wl08gacs7ndc6xr067qjn3ngx39q1jdcywz" {}; libiconv = applePackage' "libiconv" "59" "macos-11.0.1" "0lwa4brdwm4lvrdnxylzsn1yph4m7csgri2zkc4xb4xiisz32pwp" {}; +libmalloc = applePackage' "libmalloc" "317.40.8" "macos-11.0.1" "sha256-Tdhb0mq3w4Hwvp3xHB79Vr22hCOQK6h28HCsd7jvITI=" {}; libplatform = applePackage' "libplatform" "254.40.4" "macos-11.0.1" "1qf3ri0yd8b1xjln1j1gyx7ks6k3a2jhd63blyvfby75y9s7flky" {}; libpthread = applePackage' "libpthread" "454.40.3" "macos-11.0.1" "0zljbw8mpb80n1if65hhi9lkgwbgjr8vc9wvf7q1nl3mzyl35f8p" {}; libresolv = applePackage' "libresolv" "68" "macos-11.0.1" "045ahh8nvaam9whryc2f5g5xagwp7d187r80kcff82snp5p66aq1" {}; From 489ed84733d9cf19c72c879c5a0f5b55f4fb7348 Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Mon, 12 Jun 2023 00:02:02 -0400 Subject: [PATCH 4/7] darwin.Libsystem: get malloc.h from darwin.libmalloc --- .../darwin/apple-source-releases/Libsystem/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix index aa93a32c5621..d9747b6cdc06 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix @@ -1,7 +1,7 @@ { lib, stdenv, buildPackages, fetchzip , appleDerivation', xnu, Libc, Libm, libdispatch, Libinfo , dyld, Csu, architecture, libclosure, CarbonHeaders, ncurses, CommonCrypto -, copyfile, removefile, libresolvHeaders, libresolv, Libnotify, libplatform, libpthread +, copyfile, removefile, libresolvHeaders, libresolv, Libnotify, libmalloc, libplatform, libpthread , mDNSResponder, launchd, libutilHeaders, hfsHeaders, darwin-stubs , headersOnly ? false , withLibresolv ? !headersOnly @@ -57,7 +57,7 @@ appleDerivation' stdenv { ${libclosure} ${CarbonHeaders} ${libdispatch} ${ncurses.dev} \ ${CommonCrypto} ${copyfile} ${removefile} ${libresolvHeaders} \ ${Libnotify} ${libplatform} ${mDNSResponder} ${launchd} \ - ${libutilHeaders} ${libpthread} ${hfsHeaders}; do + ${libutilHeaders} ${libmalloc} ${libpthread} ${hfsHeaders}; do (cd $dep/include && find . -name '*.h' | copyHierarchy $out/include) done From d2b670291fafe771501effe1e4917d92440bf390 Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Mon, 12 Jun 2023 01:12:50 -0400 Subject: [PATCH 5/7] darwin.Libsystem: get asl.h from upstream syslog sources --- .../apple-source-releases/Libsystem/default.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix index d9747b6cdc06..7f4fd0fe518e 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildPackages, fetchzip +{ lib, stdenv, buildPackages, fetchzip, fetchFromGitHub , appleDerivation', xnu, Libc, Libm, libdispatch, Libinfo , dyld, Csu, architecture, libclosure, CarbonHeaders, ncurses, CommonCrypto , copyfile, removefile, libresolvHeaders, libresolv, Libnotify, libmalloc, libplatform, libpthread @@ -30,6 +30,15 @@ let fi ''; }; + + # Libsystem needs `asl.h` from syslog. This is the version corresponding to the 10.12 SDK + # source release, but it hasn’t changed in newer versions. + syslog.src = fetchFromGitHub { + owner = "apple-oss-distributions"; + repo = "syslog"; + rev = "syslog-349.50.5"; + hash = "sha256-tXLW/TNsluhO1X9Rv3FANyzyOe5TE/hZz0gVo7JGvHA="; + }; in appleDerivation' stdenv { dontBuild = true; @@ -63,6 +72,9 @@ appleDerivation' stdenv { (cd ${buildPackages.darwin.cctools.dev}/include/mach-o && find . -name '*.h' | copyHierarchy $out/include/mach-o) + # Copy `asl.h` from the syslog sources since it is no longer provided as part of Libc. + cp ${syslog.src}/libsystem_asl.tproj/include/asl.h $out/include + mkdir -p $out/include/os cp ${darling.src}/src/libc/os/activity.h $out/include/os From a255182f567abd87acc59206780627bccab0d10d Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Mon, 12 Jun 2023 09:11:11 -0400 Subject: [PATCH 6/7] darwin.Libsystem: get pthread headers from darwin.libpthread Some of the headers are symlinked to the root `include` in upstream Libsystem, so also symlink them there from `include/pthread`. --- .../darwin/apple-source-releases/Libsystem/default.nix | 4 ++++ .../darwin/apple-source-releases/Libsystem/headers.txt | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix index 7f4fd0fe518e..c9cc99a6550e 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/default.nix @@ -72,6 +72,10 @@ appleDerivation' stdenv { (cd ${buildPackages.darwin.cctools.dev}/include/mach-o && find . -name '*.h' | copyHierarchy $out/include/mach-o) + for header in pthread.h pthread_impl.h pthread_spis.h sched.h; do + ln -s "$out/include/pthread/$header" "$out/include/$header" + done + # Copy `asl.h` from the syslog sources since it is no longer provided as part of Libc. cp ${syslog.src}/libsystem_asl.tproj/include/asl.h $out/include diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/headers.txt b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/headers.txt index cdca44c7292c..b6e608f81ebf 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/Libsystem/headers.txt +++ b/pkgs/os-specific/darwin/apple-source-releases/Libsystem/headers.txt @@ -1381,7 +1381,6 @@ protocols/routed.h protocols/rwhod.h protocols/talkd.h protocols/timed.h -pthread.h pthread/introspection.h pthread/pthread.h pthread/pthread_impl.h @@ -1389,9 +1388,6 @@ pthread/pthread_spis.h pthread/qos.h pthread/sched.h pthread/spawn.h -pthread_impl.h -pthread_spis.h -pthread_workqueue.h pwd.h ranlib.h readpassphrase.h @@ -1414,7 +1410,6 @@ rpc/xdr.h rpcsvc/yp_prot.h rpcsvc/ypclnt.h runetype.h -sched.h search.h secure/_common.h secure/_stdio.h From 23cb6c155b89b060eb2004cd0e1951a4097a81fe Mon Sep 17 00:00:00 2001 From: Randy Eckenrode Date: Mon, 12 Jun 2023 09:13:47 -0400 Subject: [PATCH 7/7] darwin.Libc: stop vendoring headers from other packages A number of headers in Libc are being vendored from other packages. Instead of copying them from an earlier Libc, Libsystem now sources them from their respective packages (see below). This allows Libc_old to be dropped and avoids any potential clashes when building Libsystem. libmalloc: * malloc/malloc.h libplatform: * setjmp.h * ucontext.h * libkern/OSAtomic.h * libkern/OSCacheControl.h libpthread: * pthread*.h * sched.h * spawn.h syslog (vendored because only one file is needed): * asl.h xnu: * spawn.h (a different one from libpthread) * libproc.h --- .../apple-source-releases/Libc/825_40_1.nix | 13 ------------- .../apple-source-releases/Libc/default.nix | 17 +---------------- .../apple-source-releases/Libc/headers.txt | 13 ------------- .../darwin/apple-source-releases/default.nix | 1 - 4 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 pkgs/os-specific/darwin/apple-source-releases/Libc/825_40_1.nix diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libc/825_40_1.nix b/pkgs/os-specific/darwin/apple-source-releases/Libc/825_40_1.nix deleted file mode 100644 index c9202b536583..000000000000 --- a/pkgs/os-specific/darwin/apple-source-releases/Libc/825_40_1.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ appleDerivation', stdenvNoCC, ed, unifdef }: - -appleDerivation' stdenvNoCC { - nativeBuildInputs = [ ed unifdef ]; - - installPhase = '' - export SRCROOT=$PWD - export DSTROOT=$out - export PUBLIC_HEADERS_FOLDER_PATH=include - export PRIVATE_HEADERS_FOLDER_PATH=include - bash xcodescripts/headers.sh - ''; -} diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libc/default.nix b/pkgs/os-specific/darwin/apple-source-releases/Libc/default.nix index 915d3a61f816..6ff3cec2f1eb 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/Libc/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/Libc/default.nix @@ -1,4 +1,4 @@ -{ appleDerivation', stdenvNoCC, ed, unifdef, Libc_old, Libc_10-9 }: +{ appleDerivation', stdenvNoCC, ed, unifdef, Libc_10-9 }: appleDerivation' stdenvNoCC { nativeBuildInputs = [ ed unifdef ]; @@ -7,7 +7,6 @@ appleDerivation' stdenvNoCC { ./0001-Define-TARGET_OS_EMBEDDED-in-std-lib-io-if-not-defin.patch ]; - # TODO: asl.h actually comes from syslog project now installPhase = '' export SRCROOT=$PWD export DSTROOT=$out @@ -18,20 +17,6 @@ appleDerivation' stdenvNoCC { cp ${./CrashReporterClient.h} $out/include/CrashReporterClient.h cp ${Libc_10-9}/include/NSSystemDirectories.h $out/include - - # Ugh Apple stopped releasing this stuff so we need an older one... - cp ${Libc_old}/include/spawn.h $out/include - cp ${Libc_old}/include/setjmp.h $out/include - cp ${Libc_old}/include/ucontext.h $out/include - cp ${Libc_old}/include/pthread*.h $out/include - cp ${Libc_old}/include/sched.h $out/include - cp -R ${Libc_old}/include/malloc $out/include - - mkdir -p $out/include/libkern - cp ${Libc_old}/include/asl.h $out/include - cp ${Libc_old}/include/libproc.h $out/include - cp ${Libc_old}/include/libkern/OSAtomic.h $out/include/libkern - cp ${Libc_old}/include/libkern/OSCacheControl.h $out/include/libkern ''; appleHeaders = builtins.readFile ./headers.txt; diff --git a/pkgs/os-specific/darwin/apple-source-releases/Libc/headers.txt b/pkgs/os-specific/darwin/apple-source-releases/Libc/headers.txt index ea62e31dc781..c7cad6343d8a 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/Libc/headers.txt +++ b/pkgs/os-specific/darwin/apple-source-releases/Libc/headers.txt @@ -21,7 +21,6 @@ arpa/inet.h arpa/nameser_compat.h arpa/telnet.h arpa/tftp.h -asl.h assert.h bitstring.h cpio.h @@ -49,12 +48,8 @@ langinfo.h libc.h libc_private.h libgen.h -libkern/OSAtomic.h -libkern/OSCacheControl.h -libproc.h limits.h locale.h -malloc/malloc.h memory.h monetary.h monitor.h @@ -72,24 +67,17 @@ protocols/routed.h protocols/rwhod.h protocols/talkd.h protocols/timed.h -pthread.h -pthread_impl.h -pthread_spis.h -pthread_workqueue.h ranlib.h readpassphrase.h regex.h runetype.h -sched.h search.h secure/_common.h secure/_stdio.h secure/_string.h semaphore.h -setjmp.h sgtty.h signal.h -spawn.h stab.h standards.h stddef.h @@ -112,7 +100,6 @@ time.h timeconv.h ttyent.h tzfile.h -ucontext.h ulimit.h unistd.h util.h diff --git a/pkgs/os-specific/darwin/apple-source-releases/default.nix b/pkgs/os-specific/darwin/apple-source-releases/default.nix index a277ebfd580f..ab13e91e3735 100644 --- a/pkgs/os-specific/darwin/apple-source-releases/default.nix +++ b/pkgs/os-specific/darwin/apple-source-releases/default.nix @@ -266,7 +266,6 @@ developerToolsPackages_11_3_1 // macosPackages_11_0_1 // { rev = "Libc-997.90.3"; hash = "sha256-B18RNO+Rai5XE52TKdJV7eknosTZ+bRERkiU12d/kPU="; }; - Libc_old = applePackage "Libc/825_40_1.nix" "osx-10.8.5" "sha256-JvPSY7FzraGpqF/jYLfhU2o/2195NuKrXsryYfVtx3s=" {}; }; libclosure = applePackage "libclosure" "osx-10.11.6" "sha256-L5rQ+UBpf3B+W1U+gZKk7fXulslHsc8lxnCsplV+nr0=" {}; libdispatch = applePackage "libdispatch" "osx-10.10.5" "sha256-jfAEk0OLrJa9AIZVikIoHomd+l+4rCfc320Xh50qK5M=" {};