From d69c4f553c69fb0e64c281f85eb5822bcdfb4739 Mon Sep 17 00:00:00 2001 From: blyxyas Date: Sat, 10 Jun 2023 11:21:28 +0200 Subject: [PATCH 1/3] Add `SPEEDTEST` --- book/src/development/speedtest.md | 24 ++++++++++++++++ tests/compile-test.rs | 46 +++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 book/src/development/speedtest.md diff --git a/book/src/development/speedtest.md b/book/src/development/speedtest.md new file mode 100644 index 00000000000..0db718e6ad6 --- /dev/null +++ b/book/src/development/speedtest.md @@ -0,0 +1,24 @@ +# Speedtest +`SPEEDTEST` is the tool we use to measure lint's performance, it works by executing the same test several times. + +It's useful for measuring changes to current lints and deciding if the performance changes too much. `SPEEDTEST` is +accessed by the `SPEEDTEST` (and `SPEEDTEST_*`) environment variables. + +## Checking Speedtest + +To do a simple speed test of a lint (e.g. `allow_attributes`), use this command. + +```sh +$ SPEEDTEST=ui TESTNAME="allow_attributes" cargo uitest -- --nocapture +``` + +This will test all `ui` tests (`SPEEDTEST=ui`) whose names start with `allow_attributes`. By default, `SPEEDTEST` will +iterate your test 1000 times. But you can change this with `SPEEDTEST_ITERATIONS`. + +```sh +$ SPEEDTEST=toml SPEEDTEST_ITERATIONS=100 TESTNAME="semicolon_block" cargo uitest -- --nocapture +``` + +> **WARNING**: Be sure to use `-- --nocapture` at the end of the command to see the average test time. If you don't +> use `-- --nocapture` (e.g. `SPEEDTEST=ui` `TESTNAME="let_underscore_untyped" cargo uitest -- --nocapture`), this +> will not show up. diff --git a/tests/compile-test.rs b/tests/compile-test.rs index c5e8622bc85..c45339ba0b5 100644 --- a/tests/compile-test.rs +++ b/tests/compile-test.rs @@ -213,12 +213,46 @@ fn run_ui_cargo() { fn main() { set_var("CLIPPY_DISABLE_DOCS_LINKS", "true"); - run_ui(); - run_ui_toml(); - run_ui_cargo(); - run_internal_tests(); - rustfix_coverage_known_exceptions_accuracy(); - ui_cargo_toml_metadata(); + // The SPEEDTEST_* env variables can be used to check Clippy's performance on your PR. It runs the + // affected test 1000 times and gets the average. + if let Ok(speedtest) = std::env::var("SPEEDTEST") { + println!("----------- STARTING SPEEDTEST -----------"); + let f; + match speedtest.as_str() { + "ui" => f = run_ui as fn(), + "cargo" => f = run_ui_cargo as fn(), + "toml" => f = run_ui_toml as fn(), + "internal" => f = run_internal_tests as fn(), + "rustfix-coverage-known-exceptions-accuracy" => f = rustfix_coverage_known_exceptions_accuracy as fn(), + "ui-cargo-toml-metadata" => f = ui_cargo_toml_metadata() as fn(), + + _ => panic!("unknown speedtest: {speedtest} || accepted speedtests are: [ui, cargo, toml, internal]"), + } + + let iterations; + if let Ok(iterations_str) = std::env::var("SPEEDTEST_ITERATIONS") { + iterations = iterations_str + .parse::() + .unwrap_or_else(|_| panic!("Couldn't parse `{}`, please use a valid u64", iterations_str)); + } else { + iterations = 1000; + } + + let mut sum = 0; + for _ in 0..iterations { + let start = std::time::Instant::now(); + f(); + sum += start.elapsed().as_millis(); + } + println!("average {} time: {} millis.", speedtest.to_uppercase(), sum / 1000); + } else { + run_ui(); + run_ui_toml(); + run_ui_cargo(); + run_internal_tests(); + rustfix_coverage_known_exceptions_accuracy(); + ui_cargo_toml_metadata(); + } } const RUSTFIX_COVERAGE_KNOWN_EXCEPTIONS: &[&str] = &[ From 0cd8bbc8674a38dab07ce55b36339ac403f70daa Mon Sep 17 00:00:00 2001 From: blyxyas Date: Sat, 10 Jun 2023 22:43:51 +0200 Subject: [PATCH 2/3] Fix dogfood --- tests/compile-test.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/compile-test.rs b/tests/compile-test.rs index c45339ba0b5..50f8cf26fde 100644 --- a/tests/compile-test.rs +++ b/tests/compile-test.rs @@ -223,17 +223,17 @@ fn main() { "cargo" => f = run_ui_cargo as fn(), "toml" => f = run_ui_toml as fn(), "internal" => f = run_internal_tests as fn(), - "rustfix-coverage-known-exceptions-accuracy" => f = rustfix_coverage_known_exceptions_accuracy as fn(), - "ui-cargo-toml-metadata" => f = ui_cargo_toml_metadata() as fn(), + "rustfix-coverage-known-exceptions-accuracy" => f = rustfix_coverage_known_exceptions_accuracy as fn(), + "ui-cargo-toml-metadata" => f = ui_cargo_toml_metadata as fn(), _ => panic!("unknown speedtest: {speedtest} || accepted speedtests are: [ui, cargo, toml, internal]"), - } + }; let iterations; if let Ok(iterations_str) = std::env::var("SPEEDTEST_ITERATIONS") { iterations = iterations_str .parse::() - .unwrap_or_else(|_| panic!("Couldn't parse `{}`, please use a valid u64", iterations_str)); + .unwrap_or_else(|_| panic!("Couldn't parse `{iterations_str}`, please use a valid u64")); } else { iterations = 1000; } @@ -250,8 +250,8 @@ fn main() { run_ui_toml(); run_ui_cargo(); run_internal_tests(); - rustfix_coverage_known_exceptions_accuracy(); - ui_cargo_toml_metadata(); + rustfix_coverage_known_exceptions_accuracy(); + ui_cargo_toml_metadata(); } } From 57923c3720fad870b1edeed1dbe74f990b431288 Mon Sep 17 00:00:00 2001 From: blyxyas Date: Wed, 28 Jun 2023 16:03:59 +0200 Subject: [PATCH 3/3] Finally fix dogfood --- tests/compile-test.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/compile-test.rs b/tests/compile-test.rs index 50f8cf26fde..bda3662acd0 100644 --- a/tests/compile-test.rs +++ b/tests/compile-test.rs @@ -217,14 +217,13 @@ fn main() { // affected test 1000 times and gets the average. if let Ok(speedtest) = std::env::var("SPEEDTEST") { println!("----------- STARTING SPEEDTEST -----------"); - let f; - match speedtest.as_str() { - "ui" => f = run_ui as fn(), - "cargo" => f = run_ui_cargo as fn(), - "toml" => f = run_ui_toml as fn(), - "internal" => f = run_internal_tests as fn(), - "rustfix-coverage-known-exceptions-accuracy" => f = rustfix_coverage_known_exceptions_accuracy as fn(), - "ui-cargo-toml-metadata" => f = ui_cargo_toml_metadata as fn(), + let f = match speedtest.as_str() { + "ui" => run_ui as fn(), + "cargo" => run_ui_cargo as fn(), + "toml" => run_ui_toml as fn(), + "internal" => run_internal_tests as fn(), + "rustfix-coverage-known-exceptions-accuracy" => rustfix_coverage_known_exceptions_accuracy as fn(), + "ui-cargo-toml-metadata" => ui_cargo_toml_metadata as fn(), _ => panic!("unknown speedtest: {speedtest} || accepted speedtests are: [ui, cargo, toml, internal]"), };