diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 3677d0a7d36..5525a5f63e9 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -15,7 +15,6 @@ pub(crate) fn prepare(dirs: &Dirs) {
     RelPath::DOWNLOAD.ensure_exists(dirs);
     crate::tests::RAND_REPO.fetch(dirs);
     crate::tests::REGEX_REPO.fetch(dirs);
-    crate::tests::PORTABLE_SIMD_REPO.fetch(dirs);
 }
 
 pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {
diff --git a/build_system/tests.rs b/build_system/tests.rs
index 1c3e615c7ab..9efb6ed715c 100644
--- a/build_system/tests.rs
+++ b/build_system/tests.rs
@@ -130,16 +130,10 @@ pub(crate) static REGEX_REPO: GitRepo = GitRepo::github(
 
 pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex_target");
 
-pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
-    "rust-lang",
-    "portable-simd",
-    "5794c837bc605c4cd9dbb884285976dfdb293cce",
-    "a64d8fdd0ed0d9c4",
-    "portable-simd",
-);
+pub(crate) static PORTABLE_SIMD_SRC: RelPath = RelPath::BUILD.join("coretests");
 
 pub(crate) static PORTABLE_SIMD: CargoProject =
-    CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable-simd_target");
+    CargoProject::new(&PORTABLE_SIMD_SRC, "portable-simd_target");
 
 static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests");
 
@@ -221,7 +215,12 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         }
     }),
     TestCase::custom("test.portable-simd", &|runner| {
-        PORTABLE_SIMD_REPO.patch(&runner.dirs);
+        apply_patches(
+            &runner.dirs,
+            "portable-simd",
+            &runner.stdlib_source.join("library/portable-simd"),
+            &PORTABLE_SIMD_SRC.to_path(&runner.dirs),
+        );
 
         PORTABLE_SIMD.clean(&runner.dirs);
 
diff --git a/rust-toolchain b/rust-toolchain
index 3a9740fd7ab..09e436b3eed 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2024-03-30"
+channel = "nightly-2024-04-05"
 components = ["rust-src", "rustc-dev", "llvm-tools"]