From ad6432c8ef9e27a37bdb440f8deba0e3e36a58f8 Mon Sep 17 00:00:00 2001 From: klensy Date: Tue, 23 Jan 2024 11:12:24 +0300 Subject: [PATCH 1/4] compiletest: reduce useless regex rebuilds before: ==8812== Total: 2,374,977,159 bytes in 6,840,026 blocks ==8812== At t-gmax: 8,090,486 bytes in 3,389 blocks ==8812== At t-end: 3,185,454 bytes in 757 blocks ==8812== Reads: 1,873,472,286 bytes ==8812== Writes: 1,249,411,589 bytes ==11212== I refs: 6,370,244,180 after: ==18725== Total: 429,769,246 bytes in 957,259 blocks ==18725== At t-gmax: 8,058,316 bytes in 3,502 blocks ==18725== At t-end: 3,045,261 bytes in 1,097 blocks ==18725== Reads: 431,872,599 bytes ==18725== Writes: 214,738,653 bytes ==20839== I refs: 1,873,010,089 --- src/tools/compiletest/src/runtest.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 8be4def15de..ca7efea07ca 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -4315,10 +4315,11 @@ impl<'test> TestCx<'test> { let mut seen_allocs = indexmap::IndexSet::new(); // The alloc-id appears in pretty-printed allocations. - let re = + static ALLOC_ID_PP_RE: Lazy = Lazy::new(|| { Regex::new(r"╾─*a(lloc)?([0-9]+)(\+0x[0-9]+)?()?( \([0-9]+ ptr bytes\))?─*╼") - .unwrap(); - normalized = re + .unwrap() + }); + normalized = ALLOC_ID_PP_RE .replace_all(&normalized, |caps: &Captures<'_>| { // Renumber the captured index. let index = caps.get(2).unwrap().as_str().to_string(); @@ -4331,8 +4332,9 @@ impl<'test> TestCx<'test> { .into_owned(); // The alloc-id appears in a sentence. - let re = Regex::new(r"\balloc([0-9]+)\b").unwrap(); - normalized = re + static ALLOC_ID_RE: Lazy = + Lazy::new(|| Regex::new(r"\balloc([0-9]+)\b").unwrap()); + normalized = ALLOC_ID_RE .replace_all(&normalized, |caps: &Captures<'_>| { let index = caps.get(1).unwrap().as_str().to_string(); let (index, _) = seen_allocs.insert_full(index); From e78d6859f7001b859bf7bacdc87c8da31f2f89e4 Mon Sep 17 00:00:00 2001 From: klensy Date: Tue, 23 Jan 2024 11:46:19 +0300 Subject: [PATCH 2/4] reduce bufreader size from default(8kb) to 1kb Headers WAY less than 1kb anyway, so this can be improved more? before ==18725== Total: 429,769,246 bytes in 957,259 blocks ==18725== At t-gmax: 8,058,316 bytes in 3,502 blocks ==18725== At t-end: 3,045,261 bytes in 1,097 blocks ==18725== Reads: 431,872,599 bytes ==18725== Writes: 214,738,653 bytes after ==49344== Total: 201,418,575 bytes in 957,174 blocks ==49344== At t-gmax: 7,937,250 bytes in 3,310 blocks ==49344== At t-end: 3,035,637 bytes in 1,076 blocks ==49344== Reads: 431,607,448 bytes ==49344== Writes: 210,731,540 bytes --- src/tools/compiletest/src/header.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index e70e01e8757..7c65e37119d 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -650,7 +650,7 @@ fn iter_header_extra( let comment = if testfile.extension().is_some_and(|e| e == "rs") { "//" } else { "#" }; - let mut rdr = BufReader::new(rdr); + let mut rdr = BufReader::with_capacity(1024, rdr); let mut ln = String::new(); let mut line_number = 0; From 0710ebb999dd633bd536e66fcaab3d2a4b1739e8 Mon Sep 17 00:00:00 2001 From: klensy Date: Tue, 23 Jan 2024 19:36:07 +0300 Subject: [PATCH 3/4] don't collect found paths into BTreeSet: keeping order of inserted Paths having high cost on hot path, collect into HashSet instead and sort afterward. from 1,858,963,938 to 1,448,975,825 I refs. --- src/tools/compiletest/src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index 60dd15841b7..543304694f6 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -25,7 +25,7 @@ use build_helper::git::{get_git_modified_files, get_git_untracked_files}; use core::panic; use getopts::Options; use lazycell::AtomicLazyCell; -use std::collections::BTreeSet; +use std::collections::HashSet; use std::ffi::OsString; use std::fs; use std::io::{self, ErrorKind}; @@ -415,7 +415,7 @@ pub fn run_tests(config: Arc) { let mut tests = Vec::new(); for c in configs { - let mut found_paths = BTreeSet::new(); + let mut found_paths = HashSet::new(); make_tests(c, &mut tests, &mut found_paths); check_overlapping_tests(&found_paths); } @@ -550,7 +550,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts { pub fn make_tests( config: Arc, tests: &mut Vec, - found_paths: &mut BTreeSet, + found_paths: &mut HashSet, ) { debug!("making tests from {:?}", config.src_base.display()); let inputs = common_inputs_stamp(&config); @@ -646,7 +646,7 @@ fn collect_tests_from_dir( relative_dir_path: &Path, inputs: &Stamp, tests: &mut Vec, - found_paths: &mut BTreeSet, + found_paths: &mut HashSet, modified_tests: &Vec, poisoned: &mut bool, ) -> io::Result<()> { @@ -1128,7 +1128,7 @@ fn not_a_digit(c: char) -> bool { !c.is_digit(10) } -fn check_overlapping_tests(found_paths: &BTreeSet) { +fn check_overlapping_tests(found_paths: &HashSet) { let mut collisions = Vec::new(); for path in found_paths { for ancestor in path.ancestors().skip(1) { @@ -1138,6 +1138,7 @@ fn check_overlapping_tests(found_paths: &BTreeSet) { } } if !collisions.is_empty() { + collisions.sort(); let collisions: String = collisions .into_iter() .map(|(path, check_parent)| format!("test {path:?} clashes with {check_parent:?}\n")) From bcfdf3307bcc0246dbe6ac709ed695a0a809c2ea Mon Sep 17 00:00:00 2001 From: klensy Date: Wed, 24 Jan 2024 14:58:59 +0300 Subject: [PATCH 4/4] add fixme about walking tests tree --- src/tools/compiletest/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index 543304694f6..667358b1a6e 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -675,6 +675,8 @@ fn collect_tests_from_dir( // Add each `.rs` file as a test, and recurse further on any // subdirectories we find, except for `aux` directories. + // FIXME: this walks full tests tree, even if we have something to ignore + // use walkdir/ignore like in tidy? for file in fs::read_dir(dir)? { let file = file?; let file_path = file.path();