Auto merge of #109216 - martingms:unicode-case-lut-shrink, r=Mark-Simulacrum

Shrink unicode case-mapping LUTs by 24k

I was looking into the binary bloat of a small program using `str::to_lowercase` and `str::to_uppercase`, and noticed that the lookup tables used for case mapping had a lot of zero-bytes in them. The reason for this is that since some characters map to up to three other characters when lower or uppercased, the LUTs store a `[char; 3]` for each character. However, the vast majority of cases only map to a single new character, in other words most of the entries are e.g. `(lowerc, [upperc, '\0', '\0'])`.
This PR introduces a new encoding scheme for these tables.

The changes reduces the size of my test binary by about 24K.

I've also done some `#[bench]`marks on unicode-heavy test data, and found that the performance of both `str::to_lowercase` and `str::to_uppercase` improves by up to 20%. These measurements are obviously very dependent on the character distribution of the data.

Someone else will have to decide whether this more complex scheme is worth it or not, I was just goofing around a bit and here's what came out of it 🤷‍♂️ No hard feelings if this isn't wanted!
This commit is contained in:
bors 2023-03-24 10:33:42 +00:00
commit f421586eed
2 changed files with 853 additions and 1830 deletions

File diff suppressed because it is too large Load Diff

View File

@ -1,36 +1,62 @@
use crate::{fmt_list, UnicodeData};
use std::fmt;
use std::{
char,
collections::BTreeMap,
fmt::{self, Write},
};
const INDEX_MASK: u32 = 1 << 22;
pub(crate) fn generate_case_mapping(data: &UnicodeData) -> String {
let mut file = String::new();
file.push_str(HEADER.trim_start());
let decl_type = "&[(char, [char; 3])]";
file.push_str(&format!(
"static LOWERCASE_TABLE: {} = &[{}];",
decl_type,
fmt_list(data.to_lower.iter().map(to_mapping))
));
write!(file, "const INDEX_MASK: u32 = 0x{:x};", INDEX_MASK).unwrap();
file.push_str("\n\n");
file.push_str(&format!(
"static UPPERCASE_TABLE: {} = &[{}];",
decl_type,
fmt_list(data.to_upper.iter().map(to_mapping))
));
file.push_str(HEADER.trim_start());
file.push('\n');
file.push_str(&generate_tables("LOWER", &data.to_lower));
file.push_str("\n\n");
file.push_str(&generate_tables("UPPER", &data.to_upper));
file
}
fn to_mapping((key, (a, b, c)): (&u32, &(u32, u32, u32))) -> (CharEscape, [CharEscape; 3]) {
(
CharEscape(std::char::from_u32(*key).unwrap()),
[
CharEscape(std::char::from_u32(*a).unwrap()),
CharEscape(std::char::from_u32(*b).unwrap()),
CharEscape(std::char::from_u32(*c).unwrap()),
],
)
fn generate_tables(case: &str, data: &BTreeMap<u32, (u32, u32, u32)>) -> String {
let mut mappings = Vec::with_capacity(data.len());
let mut multis = Vec::new();
for (&key, &(a, b, c)) in data.iter() {
let key = char::from_u32(key).unwrap();
if key.is_ascii() {
continue;
}
let value = if b == 0 && c == 0 {
a
} else {
multis.push([
CharEscape(char::from_u32(a).unwrap()),
CharEscape(char::from_u32(b).unwrap()),
CharEscape(char::from_u32(c).unwrap()),
]);
INDEX_MASK | (u32::try_from(multis.len()).unwrap() - 1)
};
mappings.push((CharEscape(key), value));
}
let mut tables = String::new();
write!(tables, "static {}CASE_TABLE: &[(char, u32)] = &[{}];", case, fmt_list(mappings))
.unwrap();
tables.push_str("\n\n");
write!(tables, "static {}CASE_TABLE_MULTI: &[[char; 3]] = &[{}];", case, fmt_list(multis))
.unwrap();
tables
}
struct CharEscape(char);
@ -46,10 +72,16 @@ pub fn to_lower(c: char) -> [char; 3] {
if c.is_ascii() {
[(c as u8).to_ascii_lowercase() as char, '\0', '\0']
} else {
match bsearch_case_table(c, LOWERCASE_TABLE) {
None => [c, '\0', '\0'],
Some(index) => LOWERCASE_TABLE[index].1,
}
LOWERCASE_TABLE
.binary_search_by(|&(key, _)| key.cmp(&c))
.map(|i| {
let u = LOWERCASE_TABLE[i].1;
char::from_u32(u).map(|c| [c, '\0', '\0']).unwrap_or_else(|| {
// SAFETY: Index comes from statically generated table
unsafe { *LOWERCASE_TABLE_MULTI.get_unchecked((u & (INDEX_MASK - 1)) as usize) }
})
})
.unwrap_or([c, '\0', '\0'])
}
}
@ -57,14 +89,16 @@ pub fn to_upper(c: char) -> [char; 3] {
if c.is_ascii() {
[(c as u8).to_ascii_uppercase() as char, '\0', '\0']
} else {
match bsearch_case_table(c, UPPERCASE_TABLE) {
None => [c, '\0', '\0'],
Some(index) => UPPERCASE_TABLE[index].1,
}
UPPERCASE_TABLE
.binary_search_by(|&(key, _)| key.cmp(&c))
.map(|i| {
let u = UPPERCASE_TABLE[i].1;
char::from_u32(u).map(|c| [c, '\0', '\0']).unwrap_or_else(|| {
// SAFETY: Index comes from statically generated table
unsafe { *UPPERCASE_TABLE_MULTI.get_unchecked((u & (INDEX_MASK - 1)) as usize) }
})
})
.unwrap_or([c, '\0', '\0'])
}
}
fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> {
table.binary_search_by(|&(key, _)| key.cmp(&c)).ok()
}
";