diff --git a/src/tools/miri/.github/workflows/ci.yml b/src/tools/miri/.github/workflows/ci.yml
index 8ced9fa86be..5e2abdde6ac 100644
--- a/src/tools/miri/.github/workflows/ci.yml
+++ b/src/tools/miri/.github/workflows/ci.yml
@@ -189,8 +189,6 @@ jobs:
fetch-depth: 256 # get a bit more of the history
- name: install josh-proxy
run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
- - name: start josh-proxy
- run: josh-proxy --local=$HOME/.cache/josh --remote=https://github.com --no-background &
- name: setup bot git name and email
run: |
git config --global user.name 'The Miri Conjob Bot'
diff --git a/src/tools/miri/CONTRIBUTING.md b/src/tools/miri/CONTRIBUTING.md
index 60ba2cd2346..4e50b8fae81 100644
--- a/src/tools/miri/CONTRIBUTING.md
+++ b/src/tools/miri/CONTRIBUTING.md
@@ -231,25 +231,16 @@ You can also directly run Miri on a Rust source file:
## Advanced topic: Syncing with the rustc repo
We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit changes between the
-rustc and Miri repositories.
+rustc and Miri repositories. You can install it as follows:
```sh
cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
-josh-proxy --local=$HOME/.cache/josh --remote=https://github.com --no-background
```
-This uses a directory `$HOME/.cache/josh` as a cache, to speed up repeated pulling/pushing.
-
-To make josh push via ssh instead of https, you can add the following to your `.gitconfig`:
-
-```toml
-[url "git@github.com:"]
- pushInsteadOf = https://github.com/
-```
+Josh will automatically be started and stopped by `./miri`.
### Importing changes from the rustc repo
-Josh needs to be running, as described above.
We assume we start on an up-to-date master branch in the Miri repo.
```sh
@@ -268,16 +259,14 @@ needed.
### Exporting changes to the rustc repo
-Keep in mind that pushing is the most complicated job that josh has to do --
-pulling just filters the rustc history, but pushing needs to construct a new
-rustc history that would filter to the given Miri history! To avoid problems, it
-is a good idea to always pull immediately before you push. In particular, you
-should never do two josh pushes without an intermediate pull; that can lead to
-duplicated commits.
+Keep in mind that pushing is the most complicated job that josh has to do -- pulling just filters
+the rustc history, but pushing needs to construct a new rustc history that would filter to the given
+Miri history! To avoid problems, it is a good idea to always pull immediately before you push. If
+you are getting strange errors, chances are you are running into [this josh
+bug](https://github.com/josh-project/josh/issues/998). In that case, please get in touch on Zulip.
-Josh needs to be running, as described above. We will use the josh proxy to push
-to your fork of rustc. Run the following in the Miri repo, assuming we are on an
-up-to-date master branch:
+We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo,
+assuming we are on an up-to-date master branch:
```sh
# Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
@@ -287,3 +276,11 @@ up-to-date master branch:
This will create a new branch called 'miri' in your fork, and the output should
include a link to create a rustc PR that will integrate those changes into the
main repository.
+
+If this fails due to authentication problems, it can help to make josh push via ssh instead of
+https. Add the following to your `.gitconfig`:
+
+```toml
+[url "git@github.com:"]
+ pushInsteadOf = https://github.com/
+```
diff --git a/src/tools/miri/miri-script/Cargo.lock b/src/tools/miri/miri-script/Cargo.lock
index cf6062d7d7f..1a22596b774 100644
--- a/src/tools/miri/miri-script/Cargo.lock
+++ b/src/tools/miri/miri-script/Cargo.lock
@@ -8,6 +8,38 @@ version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "directories"
+version = "4.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
[[package]]
name = "dunce"
version = "1.0.4"
@@ -20,6 +52,17 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+[[package]]
+name = "getrandom"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
[[package]]
name = "itertools"
version = "0.10.5"
@@ -40,6 +83,7 @@ name = "miri-script"
version = "0.1.0"
dependencies = [
"anyhow",
+ "directories",
"dunce",
"itertools",
"path_macro",
@@ -62,6 +106,44 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6e819bbd49d5939f682638fa54826bf1650abddcd65d000923de8ad63cc7d15"
+[[package]]
+name = "proc-macro2"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "redox_users"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+dependencies = [
+ "getrandom",
+ "redox_syscall",
+ "thiserror",
+]
+
[[package]]
name = "rustc_version"
version = "0.4.0"
@@ -92,6 +174,43 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
+[[package]]
+name = "syn"
+version = "2.0.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
+
[[package]]
name = "walkdir"
version = "2.3.3"
@@ -102,6 +221,12 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
[[package]]
name = "which"
version = "4.4.0"
diff --git a/src/tools/miri/miri-script/Cargo.toml b/src/tools/miri/miri-script/Cargo.toml
index c0414a2fe37..d805a94c8f5 100644
--- a/src/tools/miri/miri-script/Cargo.toml
+++ b/src/tools/miri/miri-script/Cargo.toml
@@ -20,3 +20,4 @@ anyhow = "1.0"
xshell = "0.2"
rustc_version = "0.4"
dunce = "1.0.4"
+directories = "4"
diff --git a/src/tools/miri/miri-script/src/commands.rs b/src/tools/miri/miri-script/src/commands.rs
index cadd7ade4a5..2948219ad89 100644
--- a/src/tools/miri/miri-script/src/commands.rs
+++ b/src/tools/miri/miri-script/src/commands.rs
@@ -2,6 +2,9 @@ use std::env;
use std::ffi::OsString;
use std::io::Write;
use std::ops::Not;
+use std::process;
+use std::thread;
+use std::time;
use anyhow::{anyhow, bail, Context, Result};
use path_macro::path;
@@ -14,6 +17,7 @@ use crate::Command;
/// Used for rustc syncs.
const JOSH_FILTER: &str =
":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri";
+const JOSH_PORT: &str = "42042";
impl MiriEnv {
fn build_miri_sysroot(&mut self, quiet: bool) -> Result<()> {
@@ -81,6 +85,55 @@ impl Command {
Ok(())
}
+ fn start_josh() -> Result<impl Drop> {
+ // Determine cache directory.
+ let local_dir = {
+ let user_dirs =
+ directories::ProjectDirs::from("org", "rust-lang", "miri-josh").unwrap();
+ user_dirs.cache_dir().to_owned()
+ };
+
+ // Start josh, silencing its output.
+ let mut cmd = process::Command::new("josh-proxy");
+ cmd.arg("--local").arg(local_dir);
+ cmd.arg("--remote").arg("https://github.com");
+ cmd.arg("--port").arg(JOSH_PORT);
+ cmd.arg("--no-background");
+ cmd.stdout(process::Stdio::null());
+ cmd.stderr(process::Stdio::null());
+ let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
+ // Give it some time so hopefully the port is open. (10ms was not enough.)
+ thread::sleep(time::Duration::from_millis(100));
+
+ // Create a wrapper that stops it on drop.
+ struct Josh(process::Child);
+ impl Drop for Josh {
+ fn drop(&mut self) {
+ #[cfg(unix)]
+ {
+ // Try to gracefully shut it down.
+ process::Command::new("kill")
+ .args(["-s", "INT", &self.0.id().to_string()])
+ .output()
+ .expect("failed to SIGINT josh-proxy");
+ // Sadly there is no "wait with timeout"... so we just give it some time to finish.
+ thread::sleep(time::Duration::from_millis(100));
+ // Now hopefully it is gone.
+ if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
+ return;
+ }
+ }
+ // If that didn't work (or we're not on Unix), kill it hard.
+ eprintln!(
+ "I have to kill josh-proxy the hard way, let's hope this does not break anything."
+ );
+ self.0.kill().expect("failed to SIGKILL josh-proxy");
+ }
+ }
+
+ Ok(Josh(josh))
+ }
+
pub fn exec(self) -> Result<()> {
// First, and crucially only once, run the auto-actions -- but not for all commands.
match &self {
@@ -94,8 +147,8 @@ impl Command {
| Command::Cargo { .. } => Self::auto_actions()?,
| Command::ManySeeds { .. }
| Command::Toolchain { .. }
- | Command::RustcPull { .. }
| Command::Bench { .. }
+ | Command::RustcPull { .. }
| Command::RustcPush { .. } => {}
}
// Then run the actual command.
@@ -174,6 +227,8 @@ impl Command {
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
bail!("working directory must be clean before running `./miri rustc-pull`");
}
+ // Make sure josh is running.
+ let josh = Self::start_josh()?;
// Update rust-version file. As a separate commit, since making it part of
// the merge has confused the heck out of josh in the past.
@@ -186,7 +241,7 @@ impl Command {
.context("FAILED to commit rust-version file, something went wrong")?;
// Fetch given rustc commit.
- cmd!(sh, "git fetch http://localhost:8000/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
+ cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
.run()
.map_err(|e| {
// Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
@@ -202,6 +257,8 @@ impl Command {
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
.run()
.context("FAILED to merge new commits, something went wrong")?;
+
+ drop(josh);
Ok(())
}
@@ -213,6 +270,8 @@ impl Command {
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
bail!("working directory must be clean before running `./miri rustc-push`");
}
+ // Make sure josh is running.
+ let josh = Self::start_josh()?;
// Find a repo we can do our preparation in.
if let Ok(rustc_git) = env::var("RUSTC_GIT") {
@@ -249,6 +308,8 @@ impl Command {
}
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}")
+ .ignore_stdout()
+ .ignore_stderr() // silence the "create GitHub PR" message
.run()?;
println!();
@@ -257,7 +318,7 @@ impl Command {
println!("Pushing miri changes...");
cmd!(
sh,
- "git push http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
+ "git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
)
.run()?;
println!();
@@ -265,7 +326,7 @@ impl Command {
// Do a round-trip check to make sure the push worked as expected.
cmd!(
sh,
- "git fetch http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
+ "git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
)
.ignore_stderr()
.read()?;
@@ -278,6 +339,8 @@ impl Command {
"Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:"
);
println!(" https://github.com/{github_user}/rust/pull/new/{branch}");
+
+ drop(josh);
Ok(())
}
@@ -295,6 +358,7 @@ impl Command {
bail!("expected many-seeds command to be non-empty");
};
let sh = Shell::new()?;
+ sh.set_var("MIRI_AUTO_OPS", "no"); // just in case we get recursively invoked
for seed in seed_start..seed_end {
println!("Trying seed: {seed}");
let mut miriflags = env::var_os("MIRIFLAGS").unwrap_or_default();
diff --git a/src/tools/miri/miri-script/src/main.rs b/src/tools/miri/miri-script/src/main.rs
index 849a9168028..41b82cfc472 100644
--- a/src/tools/miri/miri-script/src/main.rs
+++ b/src/tools/miri/miri-script/src/main.rs
@@ -1,3 +1,5 @@
+#![allow(clippy::needless_question_mark)]
+
mod commands;
mod util;
diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs
index d57da574315..03dd9037808 100644
--- a/src/tools/miri/src/lib.rs
+++ b/src/tools/miri/src/lib.rs
@@ -32,6 +32,7 @@
clippy::needless_return,
clippy::bool_to_int_with_if,
clippy::box_default,
+ clippy::needless_question_mark,
// We are not implementing queries here so it's fine
rustc::potential_query_instability
)]