diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index e542cd49147..ae140962e40 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -354,7 +354,7 @@ impl<'test> TestCx<'test> {
         if self.props.should_ice {
             match proc_res.status.code() {
                 Some(101) => (),
-                _ => self.fatal("expected ICE"),
+                _ => self.fatal("test no longer crashes/triggers ICE! Please annotate it and add it as test to tests/ui or wherever you see fit"),
             }
         }
 
diff --git a/src/tools/tidy/src/known_bug.rs b/src/tools/tidy/src/known_bug.rs
new file mode 100644
index 00000000000..a62556f762b
--- /dev/null
+++ b/src/tools/tidy/src/known_bug.rs
@@ -0,0 +1,17 @@
+//! Tidy check to ensure that tests inside 'tests/crashes' have a '@known-bug' directive.
+
+use crate::walk::*;
+use std::path::Path;
+
+pub fn check(filepath: &Path, bad: &mut bool) {
+    walk(filepath, |path, _is_dir| filter_not_rust(path), &mut |entry, contents| {
+        let file = entry.path();
+        if !contents.lines().any(|line| line.starts_with("//@ known-bug: ")) {
+            tidy_error!(
+                bad,
+                "{} crash/ice test does not have a \"//@ known-bug: \" directive",
+                file.display()
+            );
+        }
+    });
+}
diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs
index 57436e8d7fe..23f303276aa 100644
--- a/src/tools/tidy/src/lib.rs
+++ b/src/tools/tidy/src/lib.rs
@@ -67,6 +67,7 @@ pub mod features;
 pub mod fluent_alphabetical;
 mod fluent_used;
 pub(crate) mod iter_header;
+pub mod known_bug;
 pub mod mir_opt_tests;
 pub mod pal;
 pub mod run_make_tests;
diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index 93be4d61a9a..77691815830 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -35,6 +35,7 @@ fn main() {
     let library_path = root_path.join("library");
     let compiler_path = root_path.join("compiler");
     let librustdoc_path = src_path.join("librustdoc");
+    let crashes_path = tests_path.join("crashes");
 
     let args: Vec<String> = env::args().skip(1).collect();
     let (cfg_args, pos_args) = match args.iter().position(|arg| arg == "--") {
@@ -108,6 +109,7 @@ fn main() {
         check!(mir_opt_tests, &tests_path, bless);
         check!(rustdoc_gui_tests, &tests_path);
         check!(rustdoc_css_themes, &librustdoc_path);
+        check!(known_bug, &crashes_path);
 
         // Checks that only make sense for the compiler.
         check!(error_codes, &root_path, &[&compiler_path, &librustdoc_path], verbose);
diff --git a/tests/crashes/span_delayed_bug.rs b/tests/crashes/span_delayed_bug.rs
deleted file mode 100644
index e410f730415..00000000000
--- a/tests/crashes/span_delayed_bug.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-#![feature(rustc_attrs)]
-
-#[rustc_error(delayed_bug_from_inside_query)]
-fn main() {}