diff --git a/src/libcore/unicode/mod.rs b/src/libcore/unicode/mod.rs index b6eaf06aa7f..94a2507e26c 100644 --- a/src/libcore/unicode/mod.rs +++ b/src/libcore/unicode/mod.rs @@ -32,28 +32,3 @@ pub use unicode_data::lowercase::lookup as Lowercase; pub use unicode_data::n::lookup as N; pub use unicode_data::uppercase::lookup as Uppercase; pub use unicode_data::white_space::lookup as White_Space; - -#[inline(always)] -fn range_search( - needle: u32, - chunk_idx_map: &[u8; N], - (last_chunk_idx, last_chunk_mapping): (u16, u8), - bitset_chunk_idx: &[[u8; 16]; N1], - bitset: &[u64; N2], -) -> bool { - let bucket_idx = (needle / 64) as usize; - let chunk_map_idx = bucket_idx / 16; - let chunk_piece = bucket_idx % 16; - let chunk_idx = if chunk_map_idx >= N { - if chunk_map_idx == last_chunk_idx as usize { - last_chunk_mapping - } else { - return false; - } - } else { - chunk_idx_map[chunk_map_idx] - }; - let idx = bitset_chunk_idx[(chunk_idx as usize)][chunk_piece]; - let word = bitset[(idx as usize)]; - (word & (1 << (needle % 64) as u64)) != 0 -} diff --git a/src/libcore/unicode/unicode_data.rs b/src/libcore/unicode/unicode_data.rs index 3e90028613c..48caa21fb0a 100644 --- a/src/libcore/unicode/unicode_data.rs +++ b/src/libcore/unicode/unicode_data.rs @@ -1,618 +1,547 @@ ///! This file is generated by src/tools/unicode-table-generator; do not edit manually! -use super::range_search; + +#[inline(always)] +fn bitset_search< + const N: usize, + const CHUNK_SIZE: usize, + const N1: usize, + const CANONICAL: usize, + const CANONICALIZED: usize, +>( + needle: u32, + chunk_idx_map: &[u8; N], + bitset_chunk_idx: &[[u8; CHUNK_SIZE]; N1], + bitset_canonical: &[u64; CANONICAL], + bitset_canonicalized: &[(u8, u8); CANONICALIZED], +) -> bool { + let bucket_idx = (needle / 64) as usize; + let chunk_map_idx = bucket_idx / CHUNK_SIZE; + let chunk_piece = bucket_idx % CHUNK_SIZE; + let chunk_idx = if let Some(&v) = chunk_idx_map.get(chunk_map_idx) { + v + } else { + return false; + }; + let idx = bitset_chunk_idx[chunk_idx as usize][chunk_piece] as usize; + let word = if let Some(word) = bitset_canonical.get(idx) { + *word + } else { + let (real_idx, mapping) = bitset_canonicalized[idx - bitset_canonical.len()]; + let mut word = bitset_canonical[real_idx as usize]; + let should_invert = mapping & (1 << 6) != 0; + if should_invert { + word = !word; + } + // Lower 6 bits + let quantity = mapping & ((1 << 6) - 1); + if mapping & (1 << 7) != 0 { + // shift + word >>= quantity as u64; + } else { + word = word.rotate_left(quantity as u32); + } + word + }; + (word & (1 << (needle % 64) as u64)) != 0 +} + +fn decode_prefix_sum(short_offset_run_header: u32) -> u32 { + short_offset_run_header & ((1 << 21) - 1) +} + +fn decode_length(short_offset_run_header: u32) -> usize { + (short_offset_run_header >> 21) as usize +} + +#[inline(always)] +fn skip_search( + needle: u32, + short_offset_runs: &[u32; SOR], + offsets: &[u8; OFFSETS], +) -> bool { + // Note that this *cannot* be past the end of the array, as the last + // element is greater than std::char::MAX (the largest possible needle). + // + // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct + // location cannot be past it, so Err(idx) != length either. + // + // This means that we can avoid bounds checking for the accesses below, too. + let last_idx = + match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) { + Ok(idx) => idx + 1, + Err(idx) => idx, + }; + + let mut offset_idx = decode_length(short_offset_runs[last_idx]); + let length = if let Some(next) = short_offset_runs.get(last_idx + 1) { + decode_length(*next) - offset_idx + } else { + offsets.len() - offset_idx + }; + let prev = + last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0); + + let total = needle - prev; + let mut prefix_sum = 0; + for _ in 0..(length - 1) { + let offset = offsets[offset_idx]; + prefix_sum += offset as u32; + if prefix_sum > total { + break; + } + offset_idx += 1; + } + offset_idx % 2 == 1 +} pub const UNICODE_VERSION: (u32, u32, u32) = (13, 0, 0); #[rustfmt::skip] pub mod alphabetic { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (196, 44); - static BITSET_CHUNKS_MAP: [u8; 196] = [ - 6, 32, 10, 18, 19, 23, 21, 12, 7, 5, 0, 20, 14, 50, 50, 50, 50, 50, 50, 37, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 49, 50, 30, 8, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 17, 31, 16, 25, 24, 26, 13, 15, - 45, 27, 0, 0, 50, 11, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 50, 50, 50, 50, 50, 48, - 50, 34, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 28, 0, 0, 0, 0, 0, 29, 0, 0, 9, 0, 33, 2, 3, 0, 0, - 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 42, 50, 50, 50, - 43, 22, 50, 50, 50, 50, 41, 50, 50, 50, 50, 50, 50, 47, 0, 0, 0, 38, 0, 50, 50, 50, 50, + static SHORT_OFFSET_RUNS: [u32; 52] = [ + 706, 33559113, 868226669, 947920662, 1157637302, 1306536960, 1310732293, 1398813696, + 1449151936, 1451270141, 1455465613, 1459660301, 1468061604, 1648425216, 1658911342, + 1661009214, 1707147904, 1793132343, 1853951616, 1994464256, 2330009312, 2418090906, + 2428579840, 2439066671, 2441167872, 2443265607, 2445371392, 2447469113, 2449567296, + 2476836856, 2508295382, 2512498688, 2518790431, 2520888060, 2533473280, 2535576576, + 2556548774, 2634145792, 2682380992, 2715936768, 2720132608, 2736910640, 2875326464, + 2887952094, 2890053429, 2894253730, 2902649825, 2906847232, 2908944926, 2911043584, + 2913145675, 2916356939, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 51] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 254, 247, 39, 68], - [0, 0, 0, 0, 0, 0, 0, 0, 111, 135, 113, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 195, 205, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 254, 254, 254, 254, 254, 210, 254, 25, 136, 251, 71, 243], - [0, 0, 182, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 107, 103, 180, 254, 254, 254, 254, 254, 254, 254, 61, 0, 155, 222, 181], - [0, 148, 30, 0, 172, 226, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [48, 80, 254, 169, 206, 123, 189, 139, 95, 179, 145, 86, 211, 204, 254, 56], - [53, 0, 0, 0, 129, 17, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0], - [59, 54, 185, 203, 171, 191, 161, 117, 158, 87, 164, 118, 162, 67, 159, 23], - [62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [95, 131, 168, 105, 254, 254, 254, 82, 254, 254, 254, 254, 236, 130, 137, 120], - [101, 0, 225, 146, 151, 2, 217, 45, 144, 246, 32, 101, 0, 0, 0, 0], - [119, 253, 224, 175, 193, 254, 227, 195, 0, 0, 0, 0, 0, 0, 0, 0], - [143, 190, 91, 0, 153, 218, 24, 0, 0, 0, 0, 92, 0, 0, 66, 0], - [150, 94, 37, 85, 102, 0, 157, 0, 88, 122, 31, 46, 89, 74, 20, 0], - [154, 34, 254, 110, 0, 84, 0, 0, 0, 0, 233, 19, 216, 108, 237, 21], - [166, 42, 165, 72, 167, 177, 126, 76, 109, 16, 127, 38, 1, 192, 124, 0], - [176, 246, 234, 174, 254, 254, 254, 254, 254, 235, 140, 241, 240, 26, 228, 128], - [213, 239, 254, 77, 209, 64, 142, 238, 63, 0, 0, 0, 0, 0, 0, 0], - [225, 101, 207, 89, 98, 81, 208, 10, 232, 83, 147, 1, 188, 13, 178, 70], - [237, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254], - [253, 254, 254, 254, 254, 254, 254, 254, 254, 214, 231, 99, 79, 78, 183, 27], - [254, 6, 100, 50, 75, 90, 254, 28, 134, 0, 202, 51, 163, 43, 0, 0], - [254, 9, 75, 75, 49, 0, 0, 0, 0, 0, 69, 0, 199, 6, 195, 93], - [254, 41, 254, 8, 0, 0, 141, 33, 145, 4, 97, 0, 55, 0, 0, 0], - [254, 62, 254, 254, 254, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 121, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 242, 170, 252, 138, 245, 254, 254, 254, 254, 220, 173, 186, 212, 219, 14], - [254, 254, 15, 132, 254, 254, 254, 254, 57, 149, 254, 65, 223, 254, 249, 187], - [254, 254, 196, 114, 201, 44, 0, 0, 254, 254, 254, 254, 95, 47, 0, 0], - [254, 254, 250, 254, 194, 229, 156, 73, 230, 215, 254, 152, 246, 248, 71, 104], - [254, 254, 254, 5, 254, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 254, 37, 200, 254, 254, 254, 254, 254, 116, 0, 0, 0, 0], - [254, 254, 254, 254, 133, 246, 244, 112, 0, 184, 254, 125, 106, 221, 145, 29], - [254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 35, 0, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 101, 37, 0, 60, 65, 160, 18, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 7, 0, 0, 0, 0, 0, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 197, 254, 254, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 35, 254, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 84, 254, 254, 254], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 11, 0, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 25, 0], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 198, 115], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 40], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 96], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 125], - [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254], + static OFFSETS: [u8; 1391] = [ + 65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 0, 4, 12, 14, 5, 7, 1, 1, 1, 86, 1, 42, + 5, 1, 2, 2, 4, 1, 1, 6, 1, 1, 3, 1, 1, 1, 20, 1, 83, 1, 139, 8, 166, 1, 38, 2, 1, 6, 41, 39, + 14, 1, 1, 1, 2, 1, 2, 1, 1, 8, 27, 4, 4, 29, 11, 5, 56, 1, 7, 14, 102, 1, 8, 4, 8, 4, 3, 10, + 3, 2, 1, 16, 48, 13, 101, 24, 33, 9, 2, 4, 1, 5, 24, 2, 19, 19, 25, 7, 11, 53, 21, 1, 18, + 12, 12, 3, 7, 6, 76, 1, 16, 1, 3, 4, 15, 13, 19, 1, 8, 2, 2, 2, 22, 1, 7, 1, 1, 3, 4, 3, 8, + 2, 2, 2, 2, 1, 1, 8, 1, 4, 2, 1, 5, 12, 2, 10, 1, 4, 3, 1, 6, 4, 2, 2, 22, 1, 7, 1, 2, 1, 2, + 1, 2, 4, 5, 4, 2, 2, 2, 4, 1, 7, 4, 1, 1, 17, 6, 11, 3, 1, 9, 1, 3, 1, 22, 1, 7, 1, 2, 1, 5, + 3, 9, 1, 3, 1, 2, 3, 1, 15, 4, 21, 4, 4, 3, 1, 8, 2, 2, 2, 22, 1, 7, 1, 2, 1, 5, 3, 8, 2, 2, + 2, 2, 9, 2, 4, 2, 1, 5, 13, 1, 16, 2, 1, 6, 3, 3, 1, 4, 3, 2, 1, 1, 1, 2, 3, 2, 3, 3, 3, 12, + 4, 5, 3, 3, 1, 3, 3, 1, 6, 1, 40, 4, 1, 8, 1, 3, 1, 23, 1, 16, 3, 8, 1, 3, 1, 3, 8, 2, 1, 3, + 5, 4, 28, 4, 1, 8, 1, 3, 1, 23, 1, 10, 1, 5, 3, 8, 1, 3, 1, 3, 8, 2, 7, 1, 1, 4, 13, 2, 13, + 13, 1, 3, 1, 41, 2, 8, 1, 3, 1, 3, 1, 1, 5, 4, 7, 5, 22, 6, 1, 3, 1, 18, 3, 24, 1, 9, 1, 1, + 2, 7, 8, 6, 1, 1, 1, 8, 18, 2, 13, 58, 5, 7, 6, 1, 51, 2, 1, 1, 1, 5, 1, 24, 1, 1, 1, 19, 1, + 3, 2, 5, 1, 1, 6, 1, 14, 4, 32, 1, 63, 8, 1, 36, 4, 17, 6, 16, 1, 36, 67, 55, 1, 1, 2, 5, + 16, 64, 10, 4, 2, 38, 1, 1, 5, 1, 2, 43, 1, 0, 1, 4, 2, 7, 1, 1, 1, 4, 2, 41, 1, 4, 2, 33, + 1, 4, 2, 7, 1, 1, 1, 4, 2, 15, 1, 57, 1, 4, 2, 67, 37, 16, 16, 86, 2, 6, 3, 0, 2, 17, 1, 26, + 5, 75, 3, 11, 7, 13, 1, 6, 12, 20, 12, 20, 12, 13, 1, 3, 1, 2, 12, 52, 2, 19, 14, 1, 4, 1, + 67, 89, 7, 43, 5, 70, 10, 31, 1, 12, 4, 9, 23, 30, 2, 5, 11, 44, 4, 26, 54, 28, 4, 63, 2, + 20, 50, 1, 23, 2, 63, 52, 1, 15, 1, 7, 52, 42, 2, 4, 10, 44, 1, 11, 14, 55, 22, 3, 10, 36, + 2, 9, 7, 43, 2, 3, 41, 4, 1, 6, 1, 2, 3, 1, 5, 192, 39, 14, 11, 0, 2, 6, 2, 38, 2, 6, 2, 8, + 1, 1, 1, 1, 1, 1, 1, 31, 2, 53, 1, 7, 1, 1, 3, 3, 1, 7, 3, 4, 2, 6, 4, 13, 5, 3, 1, 7, 116, + 1, 13, 1, 16, 13, 101, 1, 4, 1, 2, 10, 1, 1, 3, 5, 6, 1, 1, 1, 1, 1, 1, 4, 1, 11, 2, 4, 5, + 5, 4, 1, 17, 41, 0, 52, 0, 47, 1, 47, 1, 133, 6, 4, 3, 2, 12, 38, 1, 1, 5, 1, 2, 56, 7, 1, + 16, 23, 9, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 32, 47, 1, 0, 3, 25, 9, 7, 5, 2, + 5, 4, 86, 6, 3, 1, 90, 1, 4, 5, 43, 1, 94, 17, 32, 48, 16, 0, 0, 64, 0, 3, 0, 67, 46, 2, 0, + 3, 16, 10, 2, 20, 47, 5, 8, 3, 113, 39, 9, 2, 103, 2, 53, 2, 9, 42, 17, 1, 33, 24, 52, 12, + 68, 1, 1, 44, 6, 3, 1, 1, 3, 10, 33, 5, 35, 13, 29, 3, 51, 1, 12, 15, 1, 16, 16, 10, 5, 1, + 55, 9, 14, 18, 23, 3, 69, 1, 1, 1, 1, 24, 3, 2, 16, 2, 4, 11, 6, 2, 6, 2, 6, 9, 7, 1, 7, 1, + 43, 1, 14, 6, 123, 21, 0, 12, 23, 4, 49, 0, 0, 2, 106, 38, 7, 12, 5, 5, 12, 1, 13, 1, 5, 1, + 1, 1, 2, 1, 2, 1, 108, 33, 0, 18, 64, 2, 54, 40, 12, 116, 5, 1, 135, 36, 26, 6, 26, 11, 89, + 3, 6, 2, 6, 2, 6, 2, 3, 35, 12, 1, 26, 1, 19, 1, 2, 1, 15, 2, 14, 34, 123, 69, 53, 0, 29, 3, + 49, 47, 32, 13, 30, 5, 43, 5, 30, 2, 36, 4, 8, 1, 5, 42, 158, 18, 36, 4, 36, 4, 40, 8, 52, + 156, 0, 9, 22, 10, 8, 152, 6, 2, 1, 1, 44, 1, 2, 3, 1, 2, 23, 10, 23, 9, 31, 65, 19, 1, 2, + 10, 22, 10, 26, 70, 56, 6, 2, 64, 4, 1, 2, 5, 8, 1, 3, 1, 29, 42, 29, 3, 29, 35, 8, 1, 28, + 27, 54, 10, 22, 10, 19, 13, 18, 110, 73, 55, 51, 13, 51, 13, 40, 0, 42, 1, 2, 3, 2, 78, 29, + 10, 1, 8, 22, 106, 21, 27, 23, 9, 70, 60, 55, 23, 25, 23, 51, 17, 4, 8, 35, 3, 1, 9, 64, 1, + 4, 9, 2, 10, 1, 1, 1, 35, 18, 1, 34, 2, 1, 6, 1, 65, 7, 1, 1, 1, 4, 1, 15, 1, 10, 7, 57, 23, + 4, 1, 8, 2, 2, 2, 22, 1, 7, 1, 2, 1, 5, 3, 8, 2, 2, 2, 2, 3, 1, 6, 1, 5, 7, 156, 66, 1, 3, + 1, 4, 20, 3, 30, 66, 2, 2, 1, 1, 184, 54, 2, 7, 25, 6, 34, 63, 1, 1, 3, 1, 59, 54, 2, 1, 71, + 27, 2, 14, 213, 57, 103, 64, 31, 8, 2, 1, 2, 8, 1, 2, 1, 30, 1, 2, 2, 2, 2, 4, 93, 8, 2, 46, + 2, 6, 1, 1, 1, 2, 27, 51, 2, 10, 17, 72, 5, 1, 34, 57, 0, 9, 1, 45, 1, 7, 1, 1, 49, 30, 2, + 22, 1, 14, 73, 7, 1, 2, 1, 44, 3, 1, 1, 2, 1, 3, 1, 1, 2, 2, 24, 6, 1, 2, 1, 37, 1, 2, 1, 4, + 1, 1, 0, 23, 185, 1, 79, 0, 102, 111, 17, 196, 0, 0, 0, 0, 0, 0, 7, 31, 113, 30, 18, 48, 16, + 4, 31, 21, 5, 19, 0, 64, 128, 75, 4, 57, 7, 17, 64, 2, 1, 1, 12, 2, 14, 0, 8, 0, 42, 9, 0, + 0, 49, 3, 17, 4, 8, 0, 0, 107, 5, 13, 3, 9, 7, 10, 4, 1, 0, 85, 1, 71, 1, 2, 2, 1, 2, 2, 2, + 4, 1, 12, 1, 1, 1, 7, 1, 65, 1, 4, 2, 8, 1, 7, 1, 28, 1, 4, 1, 5, 1, 1, 3, 7, 1, 0, 2, 25, + 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 8, 0, 7, 1, 17, 2, 7, 1, + 2, 1, 5, 213, 45, 10, 7, 16, 1, 0, 44, 0, 197, 59, 68, 3, 1, 3, 1, 0, 4, 1, 27, 1, 2, 1, 1, + 2, 1, 1, 10, 1, 4, 1, 1, 1, 1, 6, 1, 4, 1, 1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 1, 1, 2, 4, 1, 7, 1, 4, 1, 4, 1, 1, 1, 10, 1, 17, 5, 3, 1, 5, 1, 17, 0, + 26, 6, 26, 6, 26, 0, 0, 34, 0, 11, 222, 2, 0, 14, 0, 0, 0, 0, 0, 0, ]; - static BITSET: [u64; 255] = [ - 0, 1, 7, 15, 17, 31, 63, 127, 179, 511, 1023, 2047, 2191, 4079, 4087, 8191, 8319, 16384, - 65535, 131071, 262143, 4128527, 4194303, 8461767, 24870911, 67108863, 134217727, 276824575, - 335593502, 486341884, 536805376, 536870911, 553648127, 1056964608, 1073692671, 1073741823, - 1140785663, 2147483647, 4026540127, 4294934783, 8589934591, 15032387515, 64548249055, - 68191066527, 68719476735, 115913785343, 137438953215, 1095220854783, 1099511627711, - 1099511627775, 2199023190016, 2199023255551, 4398046511103, 8641373536127, 8791831609343, - 8795690369023, 8796093022207, 13198434443263, 17592186044415, 35184321757183, - 70368744112128, 88094074470339, 140737488355327, 140737488355328, 141836999983103, - 281474976710655, 281474976710656, 563017343310239, 844472174772224, 875211255709695, - 1125625028935679, 1125899906842623, 1688915364814303, 2119858418286774, 2251795522912255, - 2251799813685247, 3377704004976767, 3509778554814463, 3905461007941631, 4503595333443583, - 4503599627370495, 8796093022142464, 9006649498927104, 9007192812290047, 9007199254740991, - 15762594400829440, 17169970223906821, 17732925109967239, 18014398491652207, - 18014398509481983, 20266198323101936, 36027697507139583, 36028792723996672, - 36028792723996703, 36028792728190975, 36028797018963967, 72057594037927935, - 90071992547409919, 143851303137705983, 144053615424700415, 144115188075855868, - 144115188075855871, 288230371860938751, 297241973452963840, 301749971126844416, - 319718190147960832, 576460743713488896, 576460743847706622, 576460752303359999, - 576460752303423486, 576460752303423487, 790380184120328175, 1152640029630136575, - 1152917029519358975, 1152921504591118335, 1152921504606845055, 1152921504606846975, - 1153765996922689951, 2161727885562420159, 2251241253188403424, 2295745090394464220, - 2305570330330005503, 2305843004918726656, 2305843004919250943, 2305843009196916483, - 2305843009213693951, 3457638613854978030, 4323455298678290390, 4557642822898941951, - 4575692405780512767, 4611686017001275199, 4611686018360336384, 4611686018427322368, - 4611686018427387903, 4656722014700830719, 6843210385291930244, 6881498031078244479, - 6908521828386340863, 8935141660164089791, 8935423131384840192, 9168765891372858879, - 9169328841326329855, 9187201948305063935, 9187343239835811327, 9216616637413720063, - 9223372036854775807, 9223372041149743103, 9223372586610589696, 9223934986808197120, - 10371930679322607615, 10502394331027995967, 11078855083321979519, 11241233151490523135, - 13006395723845991295, 13258596753222922239, 13609596598936928288, 13834776580305453567, - 13907115649320091647, 14082190885810440174, 14123225865944680428, 16212958624174047247, - 16412803692974677999, 16424062692043104238, 16424062692043104239, 16424062692043243502, - 16424625641996804079, 16429129241624174575, 16717361816799141887, 16717361816799216127, - 16788293510930366511, 17005555242810474495, 17293822569102704639, 17581979622616071300, - 17870283321271910397, 17870283321406070975, 17870283321406128127, 17978369712463020031, - 18158513764145585631, 18158781978395017215, 18194542490281852927, 18410715276682199039, - 18428729675200069631, 18428729675200069632, 18433233274827440127, 18437455399478099968, - 18437736870159843328, 18437736874452713471, 18437736874454812668, 18442240474082181119, - 18444492273895866367, 18445618173802708993, 18446181192473632767, 18446216308128218879, - 18446462598732840928, 18446462598732840959, 18446462598732840960, 18446462599806582783, - 18446462615912710143, 18446462667452317695, 18446463149025525759, 18446463629525450752, - 18446463698244468735, 18446464796682337663, 18446466966713532671, 18446466996645134335, - 18446466996779287551, 18446471394825862144, 18446471394825863167, 18446480190918885375, - 18446498607738650623, 18446532967477018623, 18446602782178705022, 18446603336221163519, - 18446603336221196287, 18446638520593285119, 18446673709243564031, 18446708893632430079, - 18446740770879700992, 18446741595513422027, 18446741874686295551, 18446743249075830783, - 18446743798965862398, 18446744056529672000, 18446744060816261120, 18446744068886102015, - 18446744069414584320, 18446744069414601696, 18446744069414617087, 18446744069414649855, - 18446744069456527359, 18446744069548736512, 18446744069548802046, 18446744069683019775, - 18446744069951455231, 18446744070421282815, 18446744070446333439, 18446744070475743231, - 18446744070488326143, 18446744071553646463, 18446744071562067967, 18446744073696837631, - 18446744073701162813, 18446744073707454463, 18446744073709027328, 18446744073709355007, - 18446744073709419615, 18446744073709486080, 18446744073709520895, 18446744073709543424, - 18446744073709550079, 18446744073709550595, 18446744073709551579, 18446744073709551599, - 18446744073709551614, 18446744073709551615, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod case_ignorable { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 33); - static BITSET_CHUNKS_MAP: [u8; 125] = [ - 25, 14, 21, 30, 28, 4, 17, 23, 22, 0, 0, 16, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 6, 9, 0, 7, 11, 32, 31, 26, 29, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, - 10, 0, 8, 0, 19, 0, 12, 0, 1, + static SHORT_OFFSET_RUNS: [u32; 32] = [ + 688, 44045149, 555751186, 559947709, 794831996, 866136069, 891330581, 916497656, 920692236, + 924908318, 1122041344, 1130430973, 1193347585, 1205931300, 1231097515, 1235294255, + 1445009723, 1453399088, 1512120051, 1575040048, 1579248368, 1583443791, 1596046493, + 1612829031, 1621219840, 1642192896, 1667359024, 1688330988, 1692526800, 1696723963, + 1705902081, 1711210992, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 47, 57], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 173, 3], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 90, 136, 38], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 104, 7, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 78, 27, 0, 148, 138, 81, 44, 119], - [0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 58, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 167, 99, 77, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 48, 0, 116, 0, 0], - [0, 0, 0, 0, 0, 172, 70, 0, 0, 8, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 67, 0, 0, 24, 0, 0], - [0, 0, 0, 29, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 135, 0, 0, 0, 0, 16, 162, 46, 86, 51, 80, 13, 111], - [0, 0, 12, 0, 0, 43, 163, 92, 35, 82, 0, 71, 175, 14, 83, 131], - [0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 133, 0, 87, 0, 150, 0, 178, 75, 0, 0, 0, 0, 0, 0, 0], - [20, 5, 61, 0, 120, 0, 0, 0, 32, 156, 176, 1, 126, 91, 69, 88], - [26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [62, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0], - [66, 0, 0, 152, 72, 25, 134, 59, 102, 124, 165, 101, 0, 64, 0, 68], - [73, 33, 0, 181, 125, 85, 122, 139, 123, 100, 123, 169, 155, 54, 4, 18], - [74, 151, 36, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [106, 135, 0, 112, 177, 107, 180, 168, 0, 0, 0, 0, 0, 0, 157, 142], - [109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [113, 50, 108, 0, 0, 0, 0, 0, 0, 0, 174, 182, 182, 114, 10, 0], - [115, 0, 0, 0, 141, 5, 0, 49, 145, 34, 31, 0, 0, 0, 0, 0], - [118, 0, 42, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [143, 95, 37, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0], - [161, 0, 103, 0, 160, 11, 30, 0, 0, 0, 0, 93, 0, 0, 0, 0], - [164, 55, 155, 53, 127, 52, 2, 28, 117, 21, 128, 19, 110, 147, 129, 9], - [170, 41, 153, 6, 0, 0, 159, 39, 158, 1, 105, 0, 65, 0, 0, 0], - [171, 149, 132, 17, 98, 89, 146, 23, 140, 0, 0, 63, 127, 97, 0, 0], - [179, 182, 0, 0, 182, 182, 182, 79, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 821] = [ + 39, 1, 6, 1, 11, 1, 35, 1, 1, 1, 71, 1, 4, 1, 1, 1, 4, 1, 2, 2, 0, 192, 4, 2, 4, 1, 9, 2, + 1, 1, 251, 7, 207, 1, 5, 1, 49, 45, 1, 1, 1, 2, 1, 2, 1, 1, 44, 1, 11, 6, 10, 11, 1, 1, 35, + 1, 10, 21, 16, 1, 101, 8, 1, 10, 1, 4, 33, 1, 1, 1, 30, 27, 91, 11, 58, 11, 4, 1, 2, 1, 24, + 24, 43, 3, 119, 48, 55, 1, 1, 1, 4, 8, 4, 1, 3, 7, 10, 2, 13, 1, 15, 1, 58, 1, 4, 4, 8, 1, + 20, 2, 26, 1, 2, 2, 57, 1, 4, 2, 4, 2, 2, 3, 3, 1, 30, 2, 3, 1, 11, 2, 57, 1, 4, 5, 1, 2, 4, + 1, 20, 2, 22, 6, 1, 1, 58, 1, 2, 1, 1, 4, 8, 1, 7, 2, 11, 2, 30, 1, 61, 1, 12, 1, 50, 1, 3, + 1, 57, 3, 5, 3, 1, 4, 7, 2, 11, 2, 29, 1, 58, 1, 2, 1, 6, 1, 5, 2, 20, 2, 28, 2, 57, 2, 4, + 4, 8, 1, 20, 2, 29, 1, 72, 1, 7, 3, 1, 1, 90, 1, 2, 7, 11, 9, 98, 1, 2, 9, 9, 1, 1, 6, 74, + 2, 27, 1, 1, 1, 1, 1, 55, 14, 1, 5, 1, 2, 5, 11, 1, 36, 9, 1, 102, 4, 1, 6, 1, 2, 2, 2, 25, + 2, 4, 3, 16, 4, 13, 1, 2, 2, 6, 1, 15, 1, 94, 1, 0, 3, 0, 3, 29, 3, 29, 2, 30, 2, 64, 2, 1, + 7, 8, 1, 2, 11, 3, 1, 5, 1, 45, 4, 52, 1, 65, 2, 34, 1, 118, 3, 4, 2, 9, 1, 6, 3, 219, 2, 2, + 1, 58, 1, 1, 7, 1, 1, 1, 1, 2, 8, 6, 10, 2, 1, 39, 1, 8, 17, 63, 4, 48, 1, 1, 5, 1, 1, 5, 1, + 40, 9, 12, 2, 32, 4, 2, 2, 1, 3, 56, 1, 1, 2, 3, 1, 1, 3, 58, 8, 2, 2, 64, 6, 82, 3, 1, 13, + 1, 7, 4, 1, 6, 1, 3, 2, 50, 63, 13, 1, 34, 95, 1, 5, 0, 1, 1, 3, 11, 3, 13, 3, 13, 3, 13, 2, + 12, 5, 8, 2, 10, 1, 2, 1, 2, 5, 49, 5, 1, 10, 1, 1, 13, 1, 16, 13, 51, 33, 0, 2, 113, 3, + 125, 1, 15, 1, 96, 32, 47, 1, 0, 1, 36, 4, 3, 5, 5, 1, 93, 6, 93, 3, 0, 1, 0, 6, 0, 1, 98, + 4, 1, 10, 1, 1, 28, 4, 80, 2, 14, 34, 78, 1, 23, 3, 109, 2, 8, 1, 3, 1, 4, 1, 25, 2, 5, 1, + 151, 2, 26, 18, 13, 1, 38, 8, 25, 11, 46, 3, 48, 1, 2, 4, 2, 2, 17, 1, 21, 2, 66, 6, 2, 2, + 2, 2, 12, 1, 8, 1, 35, 1, 11, 1, 51, 1, 1, 3, 2, 2, 5, 2, 1, 1, 27, 1, 14, 2, 5, 2, 1, 1, + 100, 5, 9, 3, 121, 1, 2, 1, 4, 1, 0, 1, 147, 16, 0, 16, 3, 1, 12, 16, 34, 1, 2, 1, 169, 1, + 7, 1, 6, 1, 11, 1, 35, 1, 1, 1, 47, 1, 45, 2, 67, 1, 21, 3, 0, 1, 226, 1, 149, 5, 0, 3, 1, + 2, 5, 4, 40, 3, 4, 1, 165, 2, 0, 4, 0, 2, 153, 11, 176, 1, 54, 15, 56, 3, 49, 4, 2, 2, 2, 1, + 15, 1, 50, 3, 36, 5, 1, 8, 62, 1, 12, 2, 52, 9, 10, 4, 2, 1, 95, 3, 2, 1, 1, 2, 6, 1, 160, + 1, 3, 8, 21, 2, 57, 2, 3, 1, 37, 7, 3, 5, 195, 8, 2, 3, 1, 1, 23, 1, 84, 6, 1, 1, 4, 2, 1, + 2, 238, 4, 6, 2, 1, 2, 27, 2, 85, 8, 2, 1, 1, 2, 106, 1, 1, 1, 2, 6, 1, 1, 101, 3, 2, 4, 1, + 5, 0, 9, 1, 2, 0, 2, 1, 1, 4, 1, 144, 4, 2, 2, 4, 1, 32, 10, 40, 6, 2, 4, 8, 1, 9, 6, 2, 3, + 46, 13, 1, 2, 0, 7, 1, 6, 1, 1, 82, 22, 2, 7, 1, 2, 1, 2, 122, 6, 3, 1, 1, 2, 1, 7, 1, 1, + 72, 2, 3, 1, 1, 1, 0, 2, 0, 9, 0, 5, 59, 7, 9, 4, 0, 1, 63, 17, 64, 2, 1, 2, 0, 2, 1, 4, 0, + 3, 9, 16, 2, 7, 30, 4, 148, 3, 0, 55, 4, 50, 8, 1, 14, 1, 22, 5, 1, 15, 0, 7, 1, 17, 2, 7, + 1, 2, 1, 5, 0, 14, 0, 4, 0, 7, 109, 8, 0, 5, 0, 1, 30, 96, 128, 240, 0, ]; - static BITSET: [u64; 183] = [ - 0, 1, 2, 3, 4, 8, 13, 15, 28, 64, 176, 191, 1016, 1792, 2047, 4080, 4096, 8192, 8193, - 16192, 30720, 32704, 32768, 40448, 131008, 262016, 2097152, 2359296, 6030336, 8323072, - 10682368, 58719232, 159383552, 234881024, 243138688, 402587711, 536805376, 536879204, - 546307648, 805306369, 1073741824, 1073741916, 2113929216, 2181038080, 3221225472, - 3758096384, 4026531840, 4294934528, 4294967296, 4512022528, 5368709120, 17179869183, - 51539615774, 51539619904, 51545907230, 51545914817, 66035122176, 115964116992, 412316860416, - 412316893184, 1030792151040, 2199023255648, 8641373536127, 8763880767488, 15397323538432, - 17303886364672, 18004502906948, 26388279066624, 36421322670080, 65128884076547, - 65970697670631, 68168642985984, 70093866270720, 70368739983360, 136957967529984, - 140737488355328, 263882790666240, 281470547525648, 281470682333183, 281474976710655, - 281474976710656, 281474976710657, 281479271675905, 562675075514368, 562949953355776, - 563001509683710, 844424930131968, 985162418487296, 1023920203366400, 2251799813685248, - 3377699721314304, 4494803534348292, 4503599627370678, 6755399441055744, 7881299349733376, - 8444256867844096, 8725724278030336, 8760633772212225, 8989057312882695, 9042383626829823, - 9851624185018758, 24822575045541890, 28848986089586688, 30958948903026688, - 35747322042253312, 53805701016846336, 58529202969772032, 72066390130950143, - 112767012056334336, 143833713099145216, 189151184399892480, 216172782113783808, - 220713756545974272, 288301294651703296, 302022650010533887, 504262420777140224, - 558446353793941504, 572520102629474304, 593978171557150752, 1008806350890729472, - 1009933895770046464, 1152921504606846976, 1152921504606846978, 1152921504606846982, - 1153202979583561736, 1441151880758558727, 1715871458028158991, 1729382256910270467, - 2301902359539744768, 2305843009196908767, 2305843009213693952, 2612078987781865472, - 2771965570646540291, 3458764513820540928, 3731232291276455943, 4539628424389459968, - 4589168020290535424, 4611404543450677248, 4611686018494513280, 4611686069967003678, - 4671217976001691648, 6341068275337658368, 6917775322003857411, 7421334051581067264, - 8070450532247928832, 8788774672813524990, 9205357638345293827, 9222809086901354496, - 9223372036854775808, 9223372036854775935, 9223512774343131136, 9224216320050987008, - 9224497932466651184, 9653465801268658176, 9727775195120332910, 10376293541461622786, - 11526998316797657088, 11529215046068469760, 12103423998558208000, 12699025049277956096, - 13005832773892571136, 13798747783286489088, 13832665517980123136, 13835058055282032640, - 13835058055282163729, 13951307220663664640, 17870283321406128128, 17906312118425092095, - 18158513697557839871, 18158513749097456062, 18374686479671623680, 18374686479671623682, - 18444496122186563584, 18445618173802708992, 18446462598732840960, 18446462598733004800, - 18446463148488654848, 18446726481523507200, 18446744069414584320, 18446744069414584322, - 18446744073575333888, 18446744073709027328, 18446744073709551615, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod cased { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 6); - static BITSET_CHUNKS_MAP: [u8; 123] = [ - 13, 18, 0, 0, 12, 0, 0, 9, 14, 10, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 2, 0, 16, 0, 8, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, - 0, 0, 0, 7, + static SHORT_OFFSET_RUNS: [u32; 19] = [ + 4256, 115348384, 136322176, 144711446, 163587254, 320875520, 325101120, 358656816, + 392231680, 404815649, 413205504, 421596288, 434182304, 442592832, 446813184, 451008166, + 528607488, 576844080, 582152586, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 19] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 62, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 10, 0, 50, 62, 58, 20], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 42, 44, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 31, 0, 62, 62, 62, 0, 62, 62, 62, 62, 54, 26, 27, 24], - [0, 0, 39, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 51, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 51, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 25], - [0, 22, 19, 37, 62, 62, 36, 61, 62, 62, 18, 12, 0, 30, 49, 38], - [0, 29, 9, 0, 34, 52, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [46, 55, 62, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [62, 6, 42, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [62, 56, 33, 60, 28, 57, 62, 62, 62, 62, 48, 35, 40, 45, 47, 5], - [62, 62, 59, 62, 41, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 283] = [ + 65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 195, 1, 4, 4, 208, 1, 36, 7, 2, 30, 5, + 96, 1, 42, 4, 2, 2, 2, 4, 1, 1, 6, 1, 1, 3, 1, 1, 1, 20, 1, 83, 1, 139, 8, 166, 1, 38, 9, + 41, 0, 38, 1, 1, 5, 1, 2, 43, 2, 3, 0, 86, 2, 6, 0, 9, 7, 43, 2, 3, 64, 192, 64, 0, 2, 6, 2, + 38, 2, 6, 2, 8, 1, 1, 1, 1, 1, 1, 1, 31, 2, 53, 1, 7, 1, 1, 3, 3, 1, 7, 3, 4, 2, 6, 4, 13, + 5, 3, 1, 7, 116, 1, 13, 1, 16, 13, 101, 1, 4, 1, 2, 10, 1, 1, 3, 5, 6, 1, 1, 1, 1, 1, 1, 4, + 1, 6, 4, 1, 2, 4, 5, 5, 4, 1, 17, 32, 3, 2, 0, 52, 0, 47, 1, 47, 1, 133, 6, 4, 3, 2, 12, 38, + 1, 1, 5, 1, 0, 46, 18, 30, 132, 102, 3, 4, 1, 48, 2, 9, 42, 2, 1, 3, 0, 43, 1, 13, 7, 80, 0, + 7, 12, 5, 0, 26, 6, 26, 0, 80, 96, 36, 4, 36, 0, 51, 13, 51, 0, 64, 0, 64, 0, 85, 1, 71, 1, + 2, 2, 1, 2, 2, 2, 4, 1, 12, 1, 1, 1, 7, 1, 65, 1, 4, 2, 8, 1, 7, 1, 28, 1, 4, 1, 5, 1, 1, 3, + 7, 1, 0, 2, 25, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 8, 0, 68, + 0, 26, 6, 26, 6, 26, 0, ]; - static BITSET: [u64; 63] = [ - 0, 15, 24, 511, 1023, 4087, 65535, 16253055, 134217726, 536805376, 1073741823, 4294967295, - 133143986179, 4398046511103, 36009005809663, 70368744177663, 2251799813685247, - 3509778554814463, 144115188074807295, 297241973452963840, 531424756029720572, - 576460743713488896, 576460743847706622, 1152921504591118335, 2295745090394464220, - 4557642822898941951, 4611686017001275199, 6908521828386340863, 8935141660164089791, - 9223934986808197120, 13605092999309557792, 16717361816799216127, 16717361816799223999, - 17005555242810474495, 17446871633794956420, 17870283321271910397, 17870283321406128127, - 18410715276682199039, 18428729675200069631, 18428729675200069632, 18437736874452713471, - 18446462598732840959, 18446462598732840960, 18446464797621878783, 18446466996779287551, - 18446603336221163519, 18446603336221196287, 18446741874686295551, 18446743249075830783, - 18446744056529672000, 18446744056529682432, 18446744069414584320, 18446744069414601696, - 18446744069422972927, 18446744070475743231, 18446744071562067967, 18446744073707454463, - 18446744073709419615, 18446744073709517055, 18446744073709550595, 18446744073709551599, - 18446744073709551600, 18446744073709551615, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod cc { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (0, 0); - static BITSET_CHUNKS_MAP: [u8; 0] = [ + static SHORT_OFFSET_RUNS: [u32; 1] = [ + 1114272, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 1] = [ - [1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 5] = [ + 0, 32, 95, 33, 0, ]; - static BITSET: [u64; 3] = [ - 0, 4294967295, 9223372036854775808, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod grapheme_extend { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 30); - static BITSET_CHUNKS_MAP: [u8; 123] = [ - 4, 15, 21, 27, 25, 3, 18, 23, 17, 0, 0, 14, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2, 7, 10, 0, 8, 12, 29, 28, 24, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 11, 0, 9, 0, 19, 0, 13, + static SHORT_OFFSET_RUNS: [u32; 31] = [ + 768, 2098307, 6292881, 10490717, 513808146, 518004748, 723528943, 731918378, 744531567, + 752920578, 769719070, 899743232, 903937950, 912327165, 916523521, 929107236, 954273451, + 958470191, 1180769328, 1252073203, 1315007216, 1319202639, 1327611037, 1340199269, + 1344395776, 1373757440, 1398923568, 1419895532, 1424091344, 1429078048, 1438581232, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 31] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 20, 46], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 74, 106, 31], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 66, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 87, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 37, 70, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 37, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 48, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 134, 82, 64, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 39, 0, 94, 0, 0], - [0, 0, 0, 0, 0, 133, 58, 0, 0, 5, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 55, 0, 0, 18, 0, 0], - [0, 0, 0, 21, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 71, 0, 118, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 9, 0, 0, 0, 129, 7, 26, 67, 0, 59, 140, 11, 68, 104], - [0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [12, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [13, 0, 50, 0, 96, 0, 0, 0, 27, 123, 139, 1, 100, 75, 57, 72], - [51, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0], - [54, 0, 0, 120, 61, 19, 105, 47, 85, 98, 131, 84, 0, 0, 0, 56], - [60, 28, 0, 141, 99, 45, 111, 109, 97, 83, 97, 136, 132, 44, 108, 22], - [63, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [89, 0, 0, 91, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0], - [93, 0, 0, 0, 113, 3, 0, 40, 115, 29, 24, 0, 0, 0, 0, 0], - [114, 78, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0], - [128, 0, 86, 0, 127, 8, 23, 0, 0, 0, 0, 76, 0, 0, 0, 0], - [130, 42, 122, 41, 112, 43, 2, 36, 95, 15, 101, 14, 90, 117, 102, 6], - [137, 34, 124, 4, 0, 0, 126, 32, 125, 1, 88, 0, 53, 0, 0, 0], - [138, 119, 92, 0, 81, 73, 116, 17, 110, 0, 0, 52, 112, 80, 0, 0], - [142, 143, 0, 0, 143, 143, 143, 66, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 689] = [ + 0, 112, 0, 7, 0, 45, 1, 1, 1, 2, 1, 2, 1, 1, 72, 11, 48, 21, 16, 1, 101, 7, 2, 6, 2, 2, 1, + 4, 35, 1, 30, 27, 91, 11, 58, 9, 9, 1, 24, 4, 1, 9, 1, 3, 1, 5, 43, 3, 119, 15, 1, 32, 55, + 1, 1, 1, 4, 8, 4, 1, 3, 7, 10, 2, 29, 1, 58, 1, 1, 1, 2, 4, 8, 1, 9, 1, 10, 2, 26, 1, 2, 2, + 57, 1, 4, 2, 4, 2, 2, 3, 3, 1, 30, 2, 3, 1, 11, 2, 57, 1, 4, 5, 1, 2, 4, 1, 20, 2, 22, 6, 1, + 1, 58, 1, 1, 2, 1, 4, 8, 1, 7, 3, 10, 2, 30, 1, 59, 1, 1, 1, 12, 1, 9, 1, 40, 1, 3, 1, 57, + 3, 5, 3, 1, 4, 7, 2, 11, 2, 29, 1, 58, 1, 2, 1, 2, 1, 3, 1, 5, 2, 7, 2, 11, 2, 28, 2, 57, 2, + 1, 1, 2, 4, 8, 1, 9, 1, 10, 2, 29, 1, 72, 1, 4, 1, 2, 3, 1, 1, 8, 1, 81, 1, 2, 7, 12, 8, 98, + 1, 2, 9, 11, 6, 74, 2, 27, 1, 1, 1, 1, 1, 55, 14, 1, 5, 1, 2, 5, 11, 1, 36, 9, 1, 102, 4, 1, + 6, 1, 2, 2, 2, 25, 2, 4, 3, 16, 4, 13, 1, 2, 2, 6, 1, 15, 1, 0, 3, 0, 3, 29, 3, 29, 2, 30, + 2, 64, 2, 1, 7, 8, 1, 2, 11, 9, 1, 45, 3, 119, 2, 34, 1, 118, 3, 4, 2, 9, 1, 6, 3, 219, 2, + 2, 1, 58, 1, 1, 7, 1, 1, 1, 1, 2, 8, 6, 10, 2, 1, 48, 17, 63, 4, 48, 7, 1, 1, 5, 1, 40, 9, + 12, 2, 32, 4, 2, 2, 1, 3, 56, 1, 1, 2, 3, 1, 1, 3, 58, 8, 2, 2, 152, 3, 1, 13, 1, 7, 4, 1, + 6, 1, 3, 2, 198, 58, 1, 5, 0, 1, 195, 33, 0, 3, 141, 1, 96, 32, 0, 6, 105, 2, 0, 4, 1, 10, + 32, 2, 80, 2, 0, 1, 3, 1, 4, 1, 25, 2, 5, 1, 151, 2, 26, 18, 13, 1, 38, 8, 25, 11, 46, 3, + 48, 1, 2, 4, 2, 2, 39, 1, 67, 6, 2, 2, 2, 2, 12, 1, 8, 1, 47, 1, 51, 1, 1, 3, 2, 2, 5, 2, 1, + 1, 42, 2, 8, 1, 238, 1, 2, 1, 4, 1, 0, 1, 0, 16, 16, 16, 0, 2, 0, 1, 226, 1, 149, 5, 0, 3, + 1, 2, 5, 4, 40, 3, 4, 1, 165, 2, 0, 4, 0, 2, 153, 11, 176, 1, 54, 15, 56, 3, 49, 4, 2, 2, + 69, 3, 36, 5, 1, 8, 62, 1, 12, 2, 52, 9, 10, 4, 2, 1, 95, 3, 2, 1, 1, 2, 6, 1, 160, 1, 3, 8, + 21, 2, 57, 2, 1, 1, 1, 1, 22, 1, 14, 7, 3, 5, 195, 8, 2, 3, 1, 1, 23, 1, 81, 1, 2, 6, 1, 1, + 2, 1, 1, 2, 1, 2, 235, 1, 2, 4, 6, 2, 1, 2, 27, 2, 85, 8, 2, 1, 1, 2, 106, 1, 1, 1, 2, 6, 1, + 1, 101, 3, 2, 4, 1, 5, 0, 9, 1, 2, 245, 1, 10, 2, 1, 1, 4, 1, 144, 4, 2, 2, 4, 1, 32, 10, + 40, 6, 2, 4, 8, 1, 9, 6, 2, 3, 46, 13, 1, 2, 0, 7, 1, 6, 1, 1, 82, 22, 2, 7, 1, 2, 1, 2, + 122, 6, 3, 1, 1, 2, 1, 7, 1, 1, 72, 2, 3, 1, 1, 1, 0, 2, 0, 5, 59, 7, 0, 1, 63, 4, 81, 1, 0, + 2, 0, 1, 1, 3, 4, 5, 8, 8, 2, 7, 30, 4, 148, 3, 0, 55, 4, 50, 8, 1, 14, 1, 22, 5, 1, 15, 0, + 7, 1, 17, 2, 7, 1, 2, 1, 5, 0, 7, 0, 4, 0, 7, 109, 7, 0, 96, 128, 240, 0, ]; - static BITSET: [u64; 144] = [ - 0, 1, 2, 8, 13, 28, 64, 182, 191, 1016, 2032, 2047, 4096, 14336, 16128, 32640, 32768, - 40448, 131008, 262016, 491520, 8323072, 8396801, 10682368, 58719232, 100663296, 134152192, - 159383552, 234881024, 243138688, 536879204, 537919040, 805306369, 1073741824, 1073741916, - 1610612736, 2153546752, 3221225472, 3758096384, 4294967296, 4512022528, 51545911364, - 51545914817, 51548004382, 51554295838, 51556262398, 68719476736, 137438953472, 412316860416, - 1030792151040, 2199023255648, 8641373536127, 8763880767488, 17303886364672, 18004502906948, - 26388279066624, 36421322670080, 65128884076547, 65970697670631, 67755789254656, - 69200441769984, 70093866270720, 263882790666240, 277076930199552, 281470547525648, - 281470681808895, 281474976710655, 281479271675904, 562675075514368, 562949953355776, - 844424930131968, 985162418487296, 1023920203366400, 2251799813685248, 3377699721314304, - 4494803534348292, 6755399441055744, 7881299349733376, 8444256867844096, 8725724278030336, - 8760633780600833, 8989057312882695, 9042383626829823, 9851624185018758, 18067175067615234, - 28848986089586688, 30958948903026688, 35747322042253312, 53805701016846336, - 58529202969772032, 189151184399892480, 220713756545974272, 466122561432846339, - 504262420777140224, 558446353793941504, 572520102629474304, 1009933895770046464, - 1152921504606846982, 1152921504606851080, 1441151880758558727, 1724878657282899983, - 2301902359539744768, 2305843009196908767, 2305843009213693952, 2310337812748042240, - 3731232291276455943, 4589168020290535424, 4609293481125347328, 4611686018427387908, - 4611686069975392286, 4671217976001691648, 5764607523034234882, 6341068275337658371, - 6341349750314369024, 7421334051581067264, 8788774672813524990, 9205357638345293827, - 9222809086901354496, 9223372036854775808, 9223372036854775935, 9224497932466651184, - 9727775195120332910, 10376293541461622786, 11526998316797657088, 11959590285459062784, - 12103423998558208000, 12699165786766311424, 13005832773892571136, 13798747783286489088, - 13835058055282032640, 13835058055282163729, 13951307220663664640, 14987979559889010690, - 17872468738205286400, 17906312118425092095, 18158513697557839871, 18158513749097456062, - 18374686479671623680, 18374686479671623682, 18446462598732840960, 18446462598732972032, - 18446744056529158144, 18446744069414584320, 18446744073709551615, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod lowercase { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (122, 6); - static BITSET_CHUNKS_MAP: [u8; 118] = [ - 12, 16, 0, 0, 10, 0, 0, 11, 13, 8, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2, 1, 0, 17, 0, 9, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, + static BITSET_CHUNKS_MAP: [u8; 123] = [ + 13, 16, 0, 0, 8, 0, 0, 11, 12, 9, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 3, 1, 0, 14, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, + 0, 0, 6, ]; static BITSET_INDEX_CHUNKS: [[u8; 16]; 18] = [ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 62, 71, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 9, 0, 50, 42, 44, 28], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35], - [0, 0, 3, 0, 71, 71, 71, 0, 46, 46, 48, 46, 24, 37, 38, 23], - [0, 29, 27, 57, 39, 51, 52, 43, 41, 70, 26, 11, 0, 34, 64, 32], - [0, 40, 8, 0, 33, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [22, 13, 54, 66, 25, 15, 56, 63, 30, 19, 12, 55, 58, 61, 65, 4], - [59, 36, 46, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [59, 49, 45, 47, 18, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [67, 5, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 14, 52, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 39, 0, 47, 43, 45, 30], + [0, 0, 0, 0, 10, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26], + [0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 54, 0, 52, 52, 52, 0, 21, 21, 64, 21, 33, 24, 23, 34], + [0, 5, 71, 0, 28, 15, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 61, 31, 17, 22, 48, 49, 44, 42, 8, 32, 38, 0, 27, 13, 29], + [11, 55, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [16, 25, 21, 35, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [16, 46, 2, 20, 63, 9, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [60, 37, 51, 12, 70, 58, 18, 1, 6, 59, 68, 19, 65, 66, 3, 41], ]; - static BITSET: [u64; 72] = [ - 0, 15, 16, 511, 3063, 65535, 16253055, 134217726, 536805376, 984263338, 4294967295, - 133143986179, 274877905920, 1099509514240, 4398046445568, 17592185782272, 36009005809663, - 46912496118442, 187649984473770, 281474972516352, 2251799813685247, 2339875276368554, - 4503599560261632, 61925590106570972, 71777214282006783, 72057592964186127, - 144115188074807295, 297241973452963840, 522417556774978824, 576460743713488896, - 1152921487426978047, 1152921504590069760, 1814856824841797631, 3607524039012697088, - 4362299189061746720, 4539628424389459968, 4601013482110844927, 4611405638684049471, - 4674456033467236607, 6172933889249159850, 9223934986808197120, 10663022717737544362, - 10808545280696953514, 12261519110656315968, 12294970652241842346, 12297829382473033730, - 12297829382473034410, 12297829382473045332, 12297829382829550250, 12297829383904690175, - 12298110845996498944, 15324248332066007893, 16596095761559859497, 16717361816799215616, - 16987577794709946364, 17293822586148356092, 18158513701852807104, 18410715274543104000, - 18428729675466407935, 18446462598732840960, 18446462598732858304, 18446462598737002495, - 18446464797621878783, 18446673704966422527, 18446726481523572736, 18446739675663105535, - 18446739675663106031, 18446742974197923840, 18446744056529682432, 18446744069414584320, - 18446744073709529733, 18446744073709551615, + static BITSET_CANONICAL: [u64; 52] = [ + 0b0000000000000000000000000000000000000000000000000000000000000000, + 0b1111111111111111110000000000000000000000000011111111111111111111, + 0b1010101010101010101010101010101010101010101010101010100000000010, + 0b1111111111111111111111000000000000000000000000001111110111111111, + 0b0000111111111111111111111111111111111111000000000000000000000000, + 0b1000000000000010000000000000000000000000000000000000000000000000, + 0b0000111111111111111111111111110000000000000000000000000011111111, + 0b0000000000000111111111111111111111111111111111111111111111111111, + 0b1111111111111111111111111111111111111111111111111010101010000101, + 0b1111111111111111111111111111111100000000000000000000000000000000, + 0b1111111111111111111111111111110000000000000000000000000000000000, + 0b1111111111111111111111110000000000000000000000000000000000000000, + 0b1111111111111111111111000000000000000000000000001111111111101111, + 0b1111111111111111111100000000000000000000000000010000000000000000, + 0b1111111111111111000000011111111111110111111111111111111111111111, + 0b1111111111111111000000000000000000000000000000000100001111000000, + 0b1111111111111111000000000000000000000000000000000000000000000000, + 0b1111111101111111111111111111111110000000000000000000000000000000, + 0b1111110000000000000000000000000011111111111111111111111111000000, + 0b1111000000000000000000000000001111110111111111111111111111111100, + 0b1010101010101010101010101010101010101010101010101101010101010100, + 0b1010101010101010101010101010101010101010101010101010101010101010, + 0b0101010110101010101010101010101010101010101010101010101010101010, + 0b0100000011011111000000001111111100000000111111110000000011111111, + 0b0011111111111111000000001111111100000000111111110000000000111111, + 0b0011111111011010000101010110001001111111111111111111111111111111, + 0b0011111100000000000000000000000000000000000000000000000000000000, + 0b0011110010001010000000000000000000000000000000000000000000100000, + 0b0011001000010000100000000000000000000000000010001100010000000000, + 0b0001100100101111101010101010101010101010111000110111111111111111, + 0b0000011101000000000000000000000000000000000000000000010100001000, + 0b0000010000100000000001000000000000000000000000000000000000000000, + 0b0000000111111111111111111111111111111111111011111111111111111111, + 0b0000000011111111000000001111111100000000001111110000000011111111, + 0b0000000011011100000000001111111100000000110011110000000011011100, + 0b0000000000001000010100000001101010101010101010101010101010101010, + 0b0000000000000000001000001011111111111111111111111111111111111111, + 0b0000000000000000000000001111111111111111110111111100000000000000, + 0b0000000000000000000000000001111100000000000000000000000000000011, + 0b0000000000000000000000000000000000111010101010101010101010101010, + 0b0000000000000000000000000000000000000000111110000000000001111111, + 0b0000000000000000000000000000000000000000000000000000101111110111, + 0b1001001111111010101010101010101010101010101010101010101010101010, + 0b1001010111111111101010101010101010101010101010101010101010101010, + 0b1010101000101001101010101010101010110101010101010101001001000000, + 0b1010101010100000100000101010101010101010101110100101000010101010, + 0b1010101010101010101010101010101011111111111111111111111111111111, + 0b1010101010101011101010101010100000000000000000000000000000000000, + 0b1101010010101010101010101010101010101010101010101010101101010101, + 0b1110011001010001001011010010101001001110001001000011000100101001, + 0b1110011111111111111111111111111111111111111111110000000000000000, + 0b1110101111000000000000000000000000001111111111111111111111111100, + ]; + static BITSET_MAPPING: [(u8, u8); 20] = [ + (0, 64), (1, 188), (1, 183), (1, 176), (1, 109), (1, 124), (1, 126), (1, 66), (1, 70), + (1, 77), (2, 146), (2, 144), (2, 83), (3, 12), (3, 6), (4, 156), (4, 78), (5, 187), + (6, 132), (7, 93), ]; pub fn lookup(c: char) -> bool { - super::range_search( + super::bitset_search( c as u32, &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, &BITSET_INDEX_CHUNKS, - &BITSET, + &BITSET_CANONICAL, + &BITSET_MAPPING, ) } } #[rustfmt::skip] pub mod n { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (127, 0); - static BITSET_CHUNKS_MAP: [u8; 127] = [ - 31, 8, 11, 25, 19, 4, 29, 21, 24, 28, 0, 16, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 13, 18, 26, 17, 23, 20, 15, 22, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 2, 0, 0, 10, 0, 14, 27, 12, 0, 1, + static SHORT_OFFSET_RUNS: [u32; 38] = [ + 1632, 18876774, 31461440, 102765417, 111154926, 115349830, 132128880, 165684320, 186656630, + 195046653, 199241735, 203436434, 216049184, 241215536, 249605104, 274792208, 278987015, + 283181793, 295766104, 320933114, 383848032, 392238160, 434181712, 442570976, 455154768, + 463544256, 476128256, 480340576, 484535936, 497144544, 501340110, 509731136, 513925872, + 518121671, 522316913, 530706688, 551681008, 556989434, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 49], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 43, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 22, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 47, 0, 0, 0, 2], - [0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 31, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 31, 0, 45, 0, 31, 0, 31, 0, 41, 0, 34], - [0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 37, 44, 4, 0, 0, 0, 0, 52, 23, 3, 0, 13], - [0, 0, 0, 7, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 35, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 62, 47, 0, 0, 0, 0, 60, 0, 0, 24, 10, 0, 5], - [0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0], - [0, 15, 0, 15, 0, 0, 0, 0, 0, 15, 0, 2, 51, 0, 0, 0], - [0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 26, 0, 0, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 11], - [0, 32, 0, 47, 65, 0, 0, 39, 0, 0, 0, 47, 0, 0, 0, 0], - [0, 46, 2, 0, 0, 71, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 59, 0, 31, 0, 42, 0, 31, 0, 15, 0, 15, 36, 0, 0, 0], - [0, 63, 30, 61, 18, 0, 55, 70, 0, 57, 20, 28, 0, 64, 29, 0], - [0, 66, 38, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 69, 19, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 9, 0], - [15, 0, 0, 0, 0, 8, 0, 17, 0, 0, 16, 0, 0, 15, 47, 0], - [40, 0, 0, 15, 2, 0, 0, 48, 0, 15, 0, 0, 0, 0, 0, 47], - [47, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [50, 0, 0, 0, 0, 0, 12, 0, 25, 21, 67, 0, 0, 0, 0, 0], - [73, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 267] = [ + 48, 10, 120, 2, 5, 1, 2, 3, 0, 10, 134, 10, 198, 10, 0, 10, 118, 10, 4, 6, 108, 10, 118, + 10, 118, 10, 2, 6, 110, 13, 115, 10, 8, 7, 103, 10, 104, 7, 7, 19, 109, 10, 96, 10, 118, 10, + 70, 20, 0, 10, 70, 10, 0, 20, 0, 3, 239, 10, 6, 10, 22, 10, 0, 10, 128, 11, 165, 10, 6, 10, + 182, 10, 86, 10, 134, 10, 6, 10, 0, 1, 3, 6, 6, 10, 198, 51, 2, 5, 0, 60, 78, 22, 0, 30, 0, + 1, 0, 1, 25, 9, 14, 3, 0, 4, 138, 10, 30, 8, 1, 15, 32, 10, 39, 15, 0, 10, 188, 10, 0, 6, + 154, 10, 38, 10, 198, 10, 22, 10, 86, 10, 0, 10, 0, 10, 0, 45, 12, 57, 17, 2, 0, 27, 36, 4, + 29, 1, 8, 1, 134, 5, 202, 10, 0, 8, 25, 7, 39, 9, 75, 5, 22, 6, 160, 2, 2, 16, 2, 46, 64, 9, + 52, 2, 30, 3, 75, 5, 104, 8, 24, 8, 41, 7, 0, 6, 48, 10, 0, 31, 158, 10, 42, 4, 112, 7, 134, + 30, 128, 10, 60, 10, 144, 10, 7, 20, 251, 10, 0, 10, 118, 10, 0, 10, 102, 10, 102, 12, 0, + 19, 93, 10, 0, 29, 227, 10, 70, 10, 0, 21, 0, 111, 0, 10, 230, 10, 1, 7, 0, 23, 0, 20, 108, + 25, 0, 50, 0, 10, 0, 10, 0, 9, 128, 10, 0, 59, 1, 3, 1, 4, 76, 45, 1, 15, 0, 13, 0, 10, 0, ]; - static BITSET: [u64; 74] = [ - 0, 999, 1023, 1026, 3072, 4064, 8191, 65408, 65472, 1048575, 1966080, 2097151, 3932160, - 4063232, 8388607, 67043328, 67044351, 134152192, 264241152, 268435455, 3758096384, - 4294901504, 17112694784, 64424509440, 549218942976, 4393751543808, 35184372023296, - 140737488355327, 272678883688448, 279275953455104, 280925220896768, 281200098803712, - 281474976448512, 492581209243648, 2251524935778304, 2251795518717952, 4503595332403200, - 4503599627370368, 8708132091985919, 9007190731849728, 17732923532771328, 71212894229889024, - 144114915328655360, 144115183780888576, 144115188075855871, 284007976623144960, - 284008251501051904, 287948901175001088, 287948901242044416, 287953294926544896, - 504407547722072192, 1152640029630136320, 1152921496016912384, 2305840810190438400, - 2305843009213693952, 3458764513820540928, 4611615649683210238, 6917529027641082367, - 8217943420044312576, 9151595642915651584, 9223372032559808512, 17870283321406128128, - 18158513697557839872, 18302628889911885824, 18374686483949813760, 18428729675200069632, - 18446181123756130304, 18446181123756131327, 18446739675663040512, 18446744069414584320, - 18446744073709355007, 18446744073709486080, 18446744073709535232, 18446744073709551615, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } #[rustfmt::skip] pub mod uppercase { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 6); - static BITSET_CHUNKS_MAP: [u8; 123] = [ - 12, 15, 0, 0, 11, 0, 0, 8, 5, 9, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 13, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 4, + static BITSET_CHUNKS_MAP: [u8; 125] = [ + 12, 15, 5, 5, 0, 5, 5, 2, 4, 11, 5, 14, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 6, 5, 13, 5, 10, 5, 5, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 7, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 5, 5, + 5, 5, 9, 5, 3, ]; static BITSET_INDEX_CHUNKS: [[u8; 16]; 17] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 9, 0, 38, 46, 44, 28], - [0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 51, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 60, 62, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 54, 0, 0, 0, 0, 0, 43, 43, 40, 43, 56, 22, 34, 35], - [0, 0, 57, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 30], - [0, 10, 0, 11, 50, 37, 36, 45, 47, 5, 0, 0, 0, 49, 17, 53], - [14, 0, 60, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [21, 52, 43, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [24, 39, 42, 41, 59, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [58, 65, 29, 16, 48, 63, 31, 19, 55, 61, 64, 32, 27, 20, 15, 3], + [41, 41, 5, 33, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 5, 0], + [41, 41, 5, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 41, 38, 41, 41, 41, 41, 41, 17, 17, 61, 17, 40, 29, 24, 23], + [41, 41, 41, 41, 9, 8, 42, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 41, 41, 41, 35, 28, 65, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 56, 41, 41, 41], + [41, 41, 41, 41, 41, 41, 41, 41, 41, 46, 41, 41, 41, 41, 41, 41], + [41, 41, 41, 41, 41, 41, 41, 41, 41, 60, 59, 41, 20, 14, 16, 4], + [41, 41, 41, 41, 47, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 41, 51, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 41, 52, 43, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [41, 53, 41, 31, 34, 21, 22, 15, 13, 32, 41, 41, 41, 11, 30, 37], + [48, 41, 9, 44, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [49, 36, 17, 27, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [50, 19, 2, 18, 10, 45, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41], + [57, 1, 26, 54, 12, 7, 25, 55, 39, 58, 6, 3, 64, 63, 62, 66], ]; - static BITSET: [u64; 67] = [ - 0, 8, 1023, 1024, 8383, 21882, 65535, 1048575, 8388607, 89478485, 134217726, 2139095039, - 4294967295, 17179869183, 1099511627775, 2199023190016, 4398046445568, 17575006099264, - 23456248059221, 70368743129088, 140737484161024, 140737488355327, 280378317225728, - 281470681743392, 281474976710655, 1169903278445909, 2251799813685247, 9007198986305536, - 9007199254741748, 17977448100528131, 18014398509481983, 288230371856744511, - 576460735123554305, 576460743713488896, 1080863910568919040, 1080897995681042176, - 1274187559846268630, 3122495741643543722, 6148633210533183488, 6148914689804861440, - 6148914690880001365, 6148914691236506283, 6148914691236516865, 6148914691236517205, - 6151773421467674709, 6184099063146390672, 7638198793012598101, 7783721355972007253, - 8863084067199903664, 9242793810247811072, 12273810184460391765, 13839347594782259332, - 13845730589451223040, 16613872850358272000, 16717361816799215616, 17293822586282573568, - 18374966856193736448, 18428729675200069632, 18442240474149289983, 18446274948748367189, - 18446462598732840960, 18446462598737035263, 18446466996779287551, 18446726481523637343, - 18446742974197924863, 18446742974197940223, 18446744069414584320, + static BITSET_CANONICAL: [u64; 41] = [ + 0b0000000000111111111111111111111111111111111111111111111111111111, + 0b1111111111111111111111110000000000000000000000000011111111111111, + 0b0101010101010101010101010101010101010101010101010101010000000001, + 0b0000011111111111111111111111110000000000000000000000000000000001, + 0b0000000000100000000000000000000000000000000000000000001011110100, + 0b1111111111111111111111111111111100000000000000000000000000000000, + 0b1111111111111111111111110000000000000000000000000000001111111111, + 0b1111111111111111111100000000000000000000000000011111110001011111, + 0b1111111111111111000000111111111111111111111111110000001111111111, + 0b1111111111111111000000000000000000000000000000000000000000000000, + 0b1111111111111110010101010101010101010101010101010101010101010101, + 0b1000000001000101000000000000000000000000000000000000000000000000, + 0b0111101100000000000000000000000000011111110111111110011110110000, + 0b0110110000000101010101010101010101010101010101010101010101010101, + 0b0110101000000000010101010101010101010101010101010101010101010101, + 0b0101010111010010010101010101010101001010101010101010010010010000, + 0b0101010101011111011111010101010101010101010001010010100001010101, + 0b0101010101010101010101010101010101010101010101010101010101010101, + 0b0101010101010101010101010101010101010101010101010010101010101011, + 0b0101010101010101010101010101010100000000000000000000000000000000, + 0b0101010101010100010101010101010000000000000000000000000000000000, + 0b0010101101010101010101010101010101010101010101010101010010101010, + 0b0001000110101110110100101101010110110001110110111100111011010110, + 0b0000111100000000000111110000000000001111000000000000111100000000, + 0b0000111100000000000000000000000000000000000000000000000000000000, + 0b0000001111111111111111111111111100000000000000000000000000111111, + 0b0000000000111111110111100110010011010000000000000000000000000011, + 0b0000000000000100001010000000010101010101010101010101010101010101, + 0b0000000000000000111111111111111100000000000000000000000000100000, + 0b0000000000000000111111110000000010101010000000000011111100000000, + 0b0000000000000000000011111111101111111111111111101101011101000000, + 0b0000000000000000000000000000000001111111011111111111111111111111, + 0b0000000000000000000000000000000000000000000000000101010101111010, + 0b0000000000000000000000000000000000000000000000000010000010111111, + 0b1010101001010101010101010101010101010101010101010101010101010101, + 0b1100000000001111001111010101000000111110001001110011100010000100, + 0b1100000000100101111010101001110100000000000000000000000000000000, + 0b1110011010010000010101010101010101010101000111001000000000000000, + 0b1110011111111111111111111111111111111111111111110000000000000000, + 0b1111000000000000000000000000001111111111111111111111111100000000, + 0b1111111100000000111111110000000000111111000000001111111100000000, + ]; + static BITSET_MAPPING: [(u8, u8); 26] = [ + (0, 182), (0, 74), (0, 166), (0, 162), (0, 159), (0, 150), (0, 148), (0, 142), (0, 135), + (0, 134), (0, 131), (0, 64), (1, 115), (1, 66), (1, 70), (1, 83), (1, 12), (1, 8), (2, 164), + (2, 146), (2, 20), (3, 146), (3, 140), (3, 134), (4, 178), (4, 171), ]; pub fn lookup(c: char) -> bool { - super::range_search( + super::bitset_search( c as u32, &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, &BITSET_INDEX_CHUNKS, - &BITSET, + &BITSET_CANONICAL, + &BITSET_MAPPING, ) } } #[rustfmt::skip] pub mod white_space { - static BITSET_LAST_CHUNK_MAP: (u16, u8) = (12, 2); - static BITSET_CHUNKS_MAP: [u8; 9] = [ - 3, 0, 0, 0, 0, 1, 0, 0, 4, + static SHORT_OFFSET_RUNS: [u32; 4] = [ + 5760, 18882560, 23080960, 40972289, ]; - static BITSET_INDEX_CHUNKS: [[u8; 16]; 5] = [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], - [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + static OFFSETS: [u8; 21] = [ + 9, 5, 18, 1, 100, 1, 26, 1, 0, 1, 0, 11, 29, 2, 5, 1, 47, 1, 0, 1, 0, ]; - static BITSET: [u64; 6] = [ - 0, 1, 2147483648, 4294967328, 4294983168, 144036023240703, - ]; - pub fn lookup(c: char) -> bool { - super::range_search( + super::skip_search( c as u32, - &BITSET_CHUNKS_MAP, - BITSET_LAST_CHUNK_MAP, - &BITSET_INDEX_CHUNKS, - &BITSET, + &SHORT_OFFSET_RUNS, + &OFFSETS, ) } } diff --git a/src/librustc_error_codes/error_codes.rs b/src/librustc_error_codes/error_codes.rs index fe68e9eacbf..2f0a3fc1d1c 100644 --- a/src/librustc_error_codes/error_codes.rs +++ b/src/librustc_error_codes/error_codes.rs @@ -389,6 +389,7 @@ E0698: include_str!("./error_codes/E0698.md"), E0699: include_str!("./error_codes/E0699.md"), E0700: include_str!("./error_codes/E0700.md"), E0701: include_str!("./error_codes/E0701.md"), +E0703: include_str!("./error_codes/E0703.md"), E0704: include_str!("./error_codes/E0704.md"), E0705: include_str!("./error_codes/E0705.md"), E0706: include_str!("./error_codes/E0706.md"), @@ -603,7 +604,6 @@ E0751: include_str!("./error_codes/E0751.md"), // E0694, // an unknown tool name found in scoped attributes E0696, // `continue` pointing to a labeled block // E0702, // replaced with a generic attribute input check - E0703, // invalid ABI // E0707, // multiple elided lifetimes used in arguments of `async fn` E0708, // `async` non-`move` closures with parameters are not currently // supported diff --git a/src/librustc_error_codes/error_codes/E0703.md b/src/librustc_error_codes/error_codes/E0703.md new file mode 100644 index 00000000000..b42677d52cb --- /dev/null +++ b/src/librustc_error_codes/error_codes/E0703.md @@ -0,0 +1,17 @@ +Invalid ABI (Application Binary Interface) used in the code. + +Erroneous code example: + +```compile_fail,E0703 +extern "invalid" fn foo() {} // error! +# fn main() {} +``` + +At present few predefined ABI's (like Rust, C, system, etc.) can be +used in Rust. Verify that the ABI is predefined. For example you can +replace the given ABI from 'Rust'. + +``` +extern "Rust" fn foo() {} // ok! +# fn main() { } +``` diff --git a/src/librustdoc/html/static/rustdoc.css b/src/librustdoc/html/static/rustdoc.css index 8887bca3c59..d091cc0c096 100644 --- a/src/librustdoc/html/static/rustdoc.css +++ b/src/librustdoc/html/static/rustdoc.css @@ -1082,8 +1082,8 @@ h3 > .collapse-toggle, h4 > .collapse-toggle { pre.rust { position: relative; - tab-width: 4; - -moz-tab-width: 4; + tab-size: 4; + -moz-tab-size: 4; } .search-failed { diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs index a41fdd2ff17..0a1b5f58815 100644 --- a/src/librustdoc/markdown.rs +++ b/src/librustdoc/markdown.rs @@ -1,4 +1,4 @@ -use std::fs::File; +use std::fs::{create_dir_all, File}; use std::io::prelude::*; use std::path::PathBuf; @@ -40,6 +40,11 @@ pub fn render( diag: &rustc_errors::Handler, edition: Edition, ) -> i32 { + if let Err(e) = create_dir_all(&options.output) { + diag.struct_err(&format!("{}: {}", options.output.display(), e)).emit(); + return 4; + } + let mut output = options.output; output.push(input.file_name().unwrap()); output.set_extension("html"); diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index 09bc183cf20..ff8f6731724 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -91,7 +91,8 @@ //! pull-requests for your suggested changes. //! //! Contributions are appreciated! If you see a part of the docs that can be -//! improved, submit a PR, or chat with us first on irc.mozilla.org #rust-docs. +//! improved, submit a PR, or chat with us first on [Discord][rust-discord] +//! #docs. //! //! # A Tour of The Rust Standard Library //! @@ -194,6 +195,7 @@ //! [multithreading]: thread/index.html //! [other]: #what-is-in-the-standard-library-documentation //! [primitive types]: ../book/ch03-02-data-types.html +//! [rust-discord]: https://discord.gg/rust-lang #![stable(feature = "rust1", since = "1.0.0")] #![doc( diff --git a/src/test/ui/codemap_tests/unicode.stderr b/src/test/ui/codemap_tests/unicode.stderr index fc2a0b5950e..01d54ac8cc8 100644 --- a/src/test/ui/codemap_tests/unicode.stderr +++ b/src/test/ui/codemap_tests/unicode.stderr @@ -8,3 +8,4 @@ LL | extern "路濫狼á́́" fn foo() {} error: aborting due to previous error +For more information about this error, try `rustc --explain E0703`. diff --git a/src/test/ui/parser/issue-8537.stderr b/src/test/ui/parser/issue-8537.stderr index b20226f87e8..a0793d94653 100644 --- a/src/test/ui/parser/issue-8537.stderr +++ b/src/test/ui/parser/issue-8537.stderr @@ -8,3 +8,4 @@ LL | "invalid-ab_isize" error: aborting due to previous error +For more information about this error, try `rustc --explain E0703`. diff --git a/src/tools/unicode-table-generator/src/main.rs b/src/tools/unicode-table-generator/src/main.rs index 839d914baa9..d5562ff91df 100644 --- a/src/tools/unicode-table-generator/src/main.rs +++ b/src/tools/unicode-table-generator/src/main.rs @@ -1,9 +1,83 @@ +//! This implements the core logic of the compression scheme used to compactly +//! encode Unicode properties. +//! +//! We have two primary goals with the encoding: we want to be compact, because +//! these tables often end up in ~every Rust program (especially the +//! grapheme_extend table, used for str debugging), including those for embedded +//! targets (where space is important). We also want to be relatively fast, +//! though this is more of a nice to have rather than a key design constraint. +//! It is expected that libraries/applications which are performance-sensitive +//! to Unicode property lookups are extremely rare, and those that care may find +//! the tradeoff of the raw bitsets worth it. For most applications, a +//! relatively fast but much smaller (and as such less cache-impacting, etc.) +//! data set is likely preferable. +//! +//! We have two separate encoding schemes: a skiplist-like approach, and a +//! compressed bitset. The datasets we consider mostly use the skiplist (it's +//! smaller) but the lowercase and uppercase sets are sufficiently sparse for +//! the bitset to be worthwhile -- for those sets the biset is a 2x size win. +//! Since the bitset is also faster, this seems an obvious choice. (As a +//! historical note, the bitset was also the prior implementation, so its +//! relative complexity had already been paid). +//! +//! ## The bitset +//! +//! The primary idea is that we 'flatten' the Unicode ranges into an enormous +//! bitset. To represent any arbitrary codepoint in a raw bitset, we would need +//! over 17 kilobytes of data per character set -- way too much for our +//! purposes. +//! +//! First, the raw bitset (one bit for every valid `char`, from 0 to 0x10FFFF, +//! not skipping the small 'gap') is associated into words (u64) and +//! deduplicated. On random data, this would be useless; on our data, this is +//! incredibly beneficial -- our data sets have (far) less than 256 unique +//! words. +//! +//! This gives us an array that maps `u8 -> word`; the current algorithm does +//! not handle the case of more than 256 unique words, but we are relatively far +//! from coming that close. +//! +//! With that scheme, we now have a single byte for every 64 codepoints. +//! +//! We further chunk these by some constant N (between 1 and 64 per group, +//! dynamically chosen for smallest size), and again deduplicate and store in an +//! array (u8 -> [u8; N]). +//! +//! The bytes of this array map into the words from the bitset above, but we +//! apply another trick here: some of these words are similar enough that they +//! can be represented by some function of another word. The particular +//! functions chosen are rotation, inversion, and shifting (right). +//! +//! ## The skiplist +//! +//! The skip list arose out of the desire for an even smaller encoding than the +//! bitset -- and was the answer to the question "what is the smallest +//! representation we can imagine?". However, it is not necessarily the +//! smallest, and if you have a better proposal, please do suggest it! +//! +//! This is a relatively straightforward encoding. First, we break up all the +//! ranges in the input data into offsets from each other, essentially a gap +//! encoding. In practice, most gaps are small -- less than u8::MAX -- so we +//! store those directly. We make use of the larger gaps (which are nicely +//! interspersed already) throughout the dataset to index this data set. +//! +//! In particular, each run of small gaps (terminating in a large gap) is +//! indexed in a separate dataset. That data set stores an index into the +//! primary offset list and a prefix sum of that offset list. These are packed +//! into a single u32 (11 bits for the offset, 21 bits for the prefix sum). +//! +//! Lookup proceeds via a binary search in the index and then a straightforward +//! linear scan (adding up the offsets) until we reach the needle, and then the +//! index of that offset is utilized as the answer to whether we're in the set +//! or not. + use std::collections::{BTreeMap, HashMap}; use std::ops::Range; use ucd_parse::Codepoints; mod case_mapping; mod raw_emitter; +mod skiplist; mod unicode_download; use raw_emitter::{emit_codepoints, RawEmitter}; @@ -152,9 +226,17 @@ fn main() { std::process::exit(1); }); + // Optional test path, which is a Rust source file testing that the unicode + // property lookups are correct. + let test_path = std::env::args().nth(2); + let unicode_data = load_data(); let ranges_by_property = &unicode_data.ranges; + if let Some(path) = test_path { + std::fs::write(&path, generate_tests(&write_location, &ranges_by_property)).unwrap(); + } + let mut total_bytes = 0; let mut modules = Vec::new(); for (property, ranges) in ranges_by_property { @@ -163,7 +245,16 @@ fn main() { emit_codepoints(&mut emitter, &ranges); modules.push((property.to_lowercase().to_string(), emitter.file)); - println!("{:15}: {} bytes, {} codepoints", property, emitter.bytes_used, datapoints,); + println!( + "{:15}: {} bytes, {} codepoints in {} ranges ({} - {}) using {}", + property, + emitter.bytes_used, + datapoints, + ranges.len(), + ranges.first().unwrap().start, + ranges.last().unwrap().end, + emitter.desc, + ); total_bytes += emitter.bytes_used; } @@ -173,7 +264,10 @@ fn main() { "///! This file is generated by src/tools/unicode-table-generator; do not edit manually!\n", ); - table_file.push_str("use super::range_search;\n\n"); + // Include the range search function + table_file.push('\n'); + table_file.push_str(include_str!("range_search.rs")); + table_file.push('\n'); table_file.push_str(&version()); @@ -236,21 +330,97 @@ fn fmt_list(values: impl IntoIterator) -> String { out } +fn generate_tests(data_path: &str, ranges: &[(&str, Vec>)]) -> String { + let mut s = String::new(); + s.push_str("#![allow(incomplete_features, unused)]\n"); + s.push_str("#![feature(const_generics)]\n\n"); + s.push_str("\n#[allow(unused)]\nuse std::hint;\n"); + s.push_str(&format!("#[path = \"{}\"]\n", data_path)); + s.push_str("mod unicode_data;\n\n"); + + s.push_str("\nfn main() {\n"); + + for (property, ranges) in ranges { + s.push_str(&format!(r#" println!("Testing {}");"#, property)); + s.push('\n'); + s.push_str(&format!(" {}_true();\n", property.to_lowercase())); + s.push_str(&format!(" {}_false();\n", property.to_lowercase())); + let mut is_true = Vec::new(); + let mut is_false = Vec::new(); + for ch_num in 0..(std::char::MAX as u32) { + if std::char::from_u32(ch_num).is_none() { + continue; + } + if ranges.iter().any(|r| r.contains(&ch_num)) { + is_true.push(ch_num); + } else { + is_false.push(ch_num); + } + } + + s.push_str(&format!(" fn {}_true() {{\n", property.to_lowercase())); + generate_asserts(&mut s, property, &is_true, true); + s.push_str(" }\n\n"); + s.push_str(&format!(" fn {}_false() {{\n", property.to_lowercase())); + generate_asserts(&mut s, property, &is_false, false); + s.push_str(" }\n\n"); + } + + s.push_str("}"); + s +} + +fn generate_asserts(s: &mut String, property: &str, points: &[u32], truthy: bool) { + for range in ranges_from_set(points) { + if range.end == range.start + 1 { + s.push_str(&format!( + " assert!({}unicode_data::{}::lookup({:?}), \"{}\");\n", + if truthy { "" } else { "!" }, + property.to_lowercase(), + std::char::from_u32(range.start).unwrap(), + range.start, + )); + } else { + s.push_str(&format!(" for chn in {:?}u32 {{\n", range)); + s.push_str(&format!( + " assert!({}unicode_data::{}::lookup(std::char::from_u32(chn).unwrap()), \"{{:?}}\", chn);\n", + if truthy { "" } else { "!" }, + property.to_lowercase(), + )); + s.push_str(" }\n"); + } + } +} + +fn ranges_from_set(set: &[u32]) -> Vec> { + let mut ranges = set.iter().map(|e| (*e)..(*e + 1)).collect::>>(); + merge_ranges(&mut ranges); + ranges +} + fn merge_ranges(ranges: &mut Vec>) { loop { let mut new_ranges = Vec::new(); let mut idx_iter = 0..(ranges.len() - 1); + let mut should_insert_last = true; while let Some(idx) = idx_iter.next() { let cur = ranges[idx].clone(); let next = ranges[idx + 1].clone(); if cur.end == next.start { - let _ = idx_iter.next(); // skip next as we're merging it in + if idx_iter.next().is_none() { + // We're merging the last element + should_insert_last = false; + } new_ranges.push(cur.start..next.end); } else { + // We're *not* merging the last element + should_insert_last = true; new_ranges.push(cur); } } - new_ranges.push(ranges.last().unwrap().clone()); + if should_insert_last { + new_ranges.push(ranges.last().unwrap().clone()); + } if new_ranges.len() == ranges.len() { *ranges = new_ranges; break; @@ -258,4 +428,12 @@ fn merge_ranges(ranges: &mut Vec>) { *ranges = new_ranges; } } + + let mut last_end = None; + for range in ranges { + if let Some(last) = last_end { + assert!(range.start > last, "{:?}", range); + } + last_end = Some(range.end); + } } diff --git a/src/tools/unicode-table-generator/src/range_search.rs b/src/tools/unicode-table-generator/src/range_search.rs new file mode 100644 index 00000000000..39b47ce703f --- /dev/null +++ b/src/tools/unicode-table-generator/src/range_search.rs @@ -0,0 +1,93 @@ +#[inline(always)] +fn bitset_search< + const N: usize, + const CHUNK_SIZE: usize, + const N1: usize, + const CANONICAL: usize, + const CANONICALIZED: usize, +>( + needle: u32, + chunk_idx_map: &[u8; N], + bitset_chunk_idx: &[[u8; CHUNK_SIZE]; N1], + bitset_canonical: &[u64; CANONICAL], + bitset_canonicalized: &[(u8, u8); CANONICALIZED], +) -> bool { + let bucket_idx = (needle / 64) as usize; + let chunk_map_idx = bucket_idx / CHUNK_SIZE; + let chunk_piece = bucket_idx % CHUNK_SIZE; + let chunk_idx = if let Some(&v) = chunk_idx_map.get(chunk_map_idx) { + v + } else { + return false; + }; + let idx = bitset_chunk_idx[chunk_idx as usize][chunk_piece] as usize; + let word = if let Some(word) = bitset_canonical.get(idx) { + *word + } else { + let (real_idx, mapping) = bitset_canonicalized[idx - bitset_canonical.len()]; + let mut word = bitset_canonical[real_idx as usize]; + let should_invert = mapping & (1 << 6) != 0; + if should_invert { + word = !word; + } + // Lower 6 bits + let quantity = mapping & ((1 << 6) - 1); + if mapping & (1 << 7) != 0 { + // shift + word >>= quantity as u64; + } else { + word = word.rotate_left(quantity as u32); + } + word + }; + (word & (1 << (needle % 64) as u64)) != 0 +} + +fn decode_prefix_sum(short_offset_run_header: u32) -> u32 { + short_offset_run_header & ((1 << 21) - 1) +} + +fn decode_length(short_offset_run_header: u32) -> usize { + (short_offset_run_header >> 21) as usize +} + +#[inline(always)] +fn skip_search( + needle: u32, + short_offset_runs: &[u32; SOR], + offsets: &[u8; OFFSETS], +) -> bool { + // Note that this *cannot* be past the end of the array, as the last + // element is greater than std::char::MAX (the largest possible needle). + // + // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct + // location cannot be past it, so Err(idx) != length either. + // + // This means that we can avoid bounds checking for the accesses below, too. + let last_idx = + match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) { + Ok(idx) => idx + 1, + Err(idx) => idx, + }; + + let mut offset_idx = decode_length(short_offset_runs[last_idx]); + let length = if let Some(next) = short_offset_runs.get(last_idx + 1) { + decode_length(*next) - offset_idx + } else { + offsets.len() - offset_idx + }; + let prev = + last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0); + + let total = needle - prev; + let mut prefix_sum = 0; + for _ in 0..(length - 1) { + let offset = offsets[offset_idx]; + prefix_sum += offset as u32; + if prefix_sum > total { + break; + } + offset_idx += 1; + } + offset_idx % 2 == 1 +} diff --git a/src/tools/unicode-table-generator/src/raw_emitter.rs b/src/tools/unicode-table-generator/src/raw_emitter.rs index 3e60ce13f92..95b63aca154 100644 --- a/src/tools/unicode-table-generator/src/raw_emitter.rs +++ b/src/tools/unicode-table-generator/src/raw_emitter.rs @@ -1,55 +1,19 @@ -//! This implements the core logic of the compression scheme used to compactly -//! encode the Unicode character classes. -//! -//! The primary idea is that we 'flatten' the Unicode ranges into an enormous -//! bitset. To represent any arbitrary codepoint in a raw bitset, we would need -//! over 17 kilobytes of data per character set -- way too much for our -//! purposes. -//! -//! We have two primary goals with the encoding: we want to be compact, because -//! these tables often end up in ~every Rust program (especially the -//! grapheme_extend table, used for str debugging), including those for embedded -//! targets (where space is important). We also want to be relatively fast, -//! though this is more of a nice to have rather than a key design constraint. -//! In practice, due to modern processor design these two are closely related. -//! -//! The encoding scheme here compresses the bitset by first deduplicating the -//! "words" (64 bits on all platforms). In practice very few words are present -//! in most data sets. -//! -//! This gives us an array that maps `u8 -> word` (if we ever went beyond 256 -//! words, we could go to u16 -> word or have some dual compression scheme -//! mapping into two separate sets; currently this is not dealt with). -//! -//! With that scheme, we now have a single byte for every 64 codepoints. We -//! further group these by 16 (arbitrarily chosen), and again deduplicate and -//! store in an array (u8 -> [u8; 16]). -//! -//! The indices into this array represent ranges of 64*16 = 1024 codepoints. -//! -//! This already reduces the top-level array to at most 1,086 bytes, but in -//! practice we usually can encode in far fewer (the first couple Unicode planes -//! are dense). -//! -//! The last byte of this top-level array is pulled out to a separate static -//! and trailing zeros are dropped; this is simply because grapheme_extend and -//! case_ignorable have a single entry in the 896th entry, so this shrinks them -//! down considerably. - use crate::fmt_list; -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::convert::TryFrom; -use std::fmt::Write; +use std::fmt::{self, Write}; use std::ops::Range; +#[derive(Clone)] pub struct RawEmitter { pub file: String, + pub desc: String, pub bytes_used: usize, } impl RawEmitter { pub fn new() -> RawEmitter { - RawEmitter { file: String::new(), bytes_used: 0 } + RawEmitter { file: String::new(), bytes_used: 0, desc: String::new() } } fn blank_line(&mut self) { @@ -59,30 +23,100 @@ impl RawEmitter { writeln!(&mut self.file, "").unwrap(); } - fn emit_bitset(&mut self, words: &[u64]) { + fn emit_bitset(&mut self, ranges: &[Range]) { + let last_code_point = ranges.last().unwrap().end; + // bitset for every bit in the codepoint range + // + // + 2 to ensure an all zero word to use for padding + let mut buckets = vec![0u64; (last_code_point as usize / 64) + 2]; + for range in ranges { + for codepoint in range.clone() { + let bucket = codepoint as usize / 64; + let bit = codepoint as u64 % 64; + buckets[bucket] |= 1 << bit; + } + } + + let mut words = buckets; + // Ensure that there's a zero word in the dataset, used for padding and + // such. + words.push(0); let unique_words = words.iter().cloned().collect::>().into_iter().collect::>(); if unique_words.len() > u8::max_value() as usize { panic!("cannot pack {} into 8 bits", unique_words.len()); } + // needed for the chunk mapping to work + assert_eq!(unique_words[0], 0, "has a zero word"); + let canonicalized = Canonicalized::canonicalize(&unique_words); - let word_indices = unique_words - .iter() - .cloned() - .enumerate() - .map(|(idx, word)| (word, u8::try_from(idx).unwrap())) - .collect::>(); + let word_indices = canonicalized.unique_mapping.clone(); + let compressed_words = words.iter().map(|w| word_indices[w]).collect::>(); - let mut idx = words.iter().map(|w| word_indices[w]).collect::>(); - let chunk_length = 16; - for _ in 0..(chunk_length - (idx.len() % chunk_length)) { - assert_eq!(unique_words[0], 0, "first word is all zeros"); - // pad out bitset index with zero words so we have all chunks of 16 - idx.push(0); + let mut best = None; + for length in 1..=64 { + let mut temp = self.clone(); + temp.emit_chunk_map(word_indices[&0], &compressed_words, length); + if let Some((_, size)) = best { + if temp.bytes_used < size { + best = Some((length, temp.bytes_used)); + } + } else { + best = Some((length, temp.bytes_used)); + } + } + self.emit_chunk_map(word_indices[&0], &compressed_words, best.unwrap().0); + + struct Bits(u64); + impl fmt::Debug for Bits { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "0b{:064b}", self.0) + } + } + + writeln!( + &mut self.file, + "static BITSET_CANONICAL: [u64; {}] = [{}];", + canonicalized.canonical_words.len(), + fmt_list(canonicalized.canonical_words.iter().map(|v| Bits(*v))), + ) + .unwrap(); + self.bytes_used += 8 * canonicalized.canonical_words.len(); + writeln!( + &mut self.file, + "static BITSET_MAPPING: [(u8, u8); {}] = [{}];", + canonicalized.canonicalized_words.len(), + fmt_list(&canonicalized.canonicalized_words), + ) + .unwrap(); + // 8 bit index into shifted words, 7 bits for shift + optional flip + // We only need it for the words that we removed by applying a shift and + // flip to them. + self.bytes_used += 2 * canonicalized.canonicalized_words.len(); + + self.blank_line(); + + writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap(); + writeln!(&mut self.file, " super::bitset_search(",).unwrap(); + writeln!(&mut self.file, " c as u32,").unwrap(); + writeln!(&mut self.file, " &BITSET_CHUNKS_MAP,").unwrap(); + writeln!(&mut self.file, " &BITSET_INDEX_CHUNKS,").unwrap(); + writeln!(&mut self.file, " &BITSET_CANONICAL,").unwrap(); + writeln!(&mut self.file, " &BITSET_MAPPING,").unwrap(); + writeln!(&mut self.file, " )").unwrap(); + writeln!(&mut self.file, "}}").unwrap(); + } + + fn emit_chunk_map(&mut self, zero_at: u8, compressed_words: &[u8], chunk_length: usize) { + let mut compressed_words = compressed_words.to_vec(); + for _ in 0..(chunk_length - (compressed_words.len() % chunk_length)) { + // pad out bitset index with zero words so we have all chunks of + // chunkchunk_length + compressed_words.push(zero_at); } let mut chunks = BTreeSet::new(); - for chunk in idx.chunks(chunk_length) { + for chunk in compressed_words.chunks(chunk_length) { chunks.insert(chunk); } let chunk_map = chunks @@ -92,23 +126,10 @@ impl RawEmitter { .map(|(idx, chunk)| (chunk, idx)) .collect::>(); let mut chunk_indices = Vec::new(); - for chunk in idx.chunks(chunk_length) { + for chunk in compressed_words.chunks(chunk_length) { chunk_indices.push(chunk_map[chunk]); } - writeln!( - &mut self.file, - "static BITSET_LAST_CHUNK_MAP: (u16, u8) = ({}, {});", - chunk_indices.len() - 1, - chunk_indices.pop().unwrap(), - ) - .unwrap(); - self.bytes_used += 3; - // Strip out the empty pieces, presuming our above pop() made us now - // have some trailing zeros. - assert_eq!(unique_words[0], 0, "first word is all zeros"); - while let Some(0) = chunk_indices.last() { - chunk_indices.pop(); - } + writeln!( &mut self.file, "static BITSET_CHUNKS_MAP: [u8; {}] = [{}];", @@ -119,52 +140,253 @@ impl RawEmitter { self.bytes_used += chunk_indices.len(); writeln!( &mut self.file, - "static BITSET_INDEX_CHUNKS: [[u8; 16]; {}] = [{}];", + "static BITSET_INDEX_CHUNKS: [[u8; {}]; {}] = [{}];", + chunk_length, chunks.len(), fmt_list(chunks.iter()), ) .unwrap(); - self.bytes_used += 16 * chunks.len(); - writeln!( - &mut self.file, - "static BITSET: [u64; {}] = [{}];", - unique_words.len(), - fmt_list(&unique_words), - ) - .unwrap(); - self.bytes_used += 8 * unique_words.len(); - } - - pub fn emit_lookup(&mut self) { - writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap(); - writeln!(&mut self.file, " super::range_search(",).unwrap(); - writeln!(&mut self.file, " c as u32,").unwrap(); - writeln!(&mut self.file, " &BITSET_CHUNKS_MAP,").unwrap(); - writeln!(&mut self.file, " BITSET_LAST_CHUNK_MAP,").unwrap(); - writeln!(&mut self.file, " &BITSET_INDEX_CHUNKS,").unwrap(); - writeln!(&mut self.file, " &BITSET,").unwrap(); - writeln!(&mut self.file, " )").unwrap(); - writeln!(&mut self.file, "}}").unwrap(); + self.bytes_used += chunk_length * chunks.len(); } } pub fn emit_codepoints(emitter: &mut RawEmitter, ranges: &[Range]) { emitter.blank_line(); - let last_code_point = ranges.last().unwrap().end; - // bitset for every bit in the codepoint range - // - // + 2 to ensure an all zero word to use for padding - let mut buckets = vec![0u64; (last_code_point as usize / 64) + 2]; - for range in ranges { - for codepoint in range.clone() { - let bucket = codepoint as usize / 64; - let bit = codepoint as u64 % 64; - buckets[bucket] |= 1 << bit; - } - } + let mut bitset = emitter.clone(); + bitset.emit_bitset(&ranges); - emitter.emit_bitset(&buckets); - emitter.blank_line(); - emitter.emit_lookup(); + let mut skiplist = emitter.clone(); + skiplist.emit_skiplist(&ranges); + + if bitset.bytes_used <= skiplist.bytes_used { + *emitter = bitset; + emitter.desc = format!("bitset"); + } else { + *emitter = skiplist; + emitter.desc = format!("skiplist"); + } +} + +struct Canonicalized { + canonical_words: Vec, + canonicalized_words: Vec<(u8, u8)>, + + /// Maps an input unique word to the associated index (u8) which is into + /// canonical_words or canonicalized_words (in order). + unique_mapping: HashMap, +} + +impl Canonicalized { + fn canonicalize(unique_words: &[u64]) -> Self { + #[derive(Copy, Clone, Debug)] + enum Mapping { + Rotate(u32), + Invert, + RotateAndInvert(u32), + ShiftRight(u32), + } + + // key is the word being mapped to + let mut mappings: BTreeMap> = BTreeMap::new(); + for &a in unique_words { + 'b: for &b in unique_words { + // skip self + if a == b { + continue; + } + + // All possible distinct rotations + for rotation in 1..64 { + if a.rotate_right(rotation) == b { + mappings.entry(b).or_default().push((a, Mapping::Rotate(rotation))); + // We're not interested in further mappings between a and b + continue 'b; + } + } + + if (!a) == b { + mappings.entry(b).or_default().push((a, Mapping::Invert)); + // We're not interested in further mappings between a and b + continue 'b; + } + + // All possible distinct rotations, inverted + for rotation in 1..64 { + if (!a.rotate_right(rotation)) == b { + mappings + .entry(b) + .or_default() + .push((a, Mapping::RotateAndInvert(rotation))); + // We're not interested in further mappings between a and b + continue 'b; + } + } + + // All possible shifts + for shift_by in 1..64 { + if a == (b >> shift_by) { + mappings + .entry(b) + .or_default() + .push((a, Mapping::ShiftRight(shift_by as u32))); + // We're not interested in further mappings between a and b + continue 'b; + } + } + } + } + // These are the bitset words which will be represented "raw" (as a u64) + let mut canonical_words = Vec::new(); + // These are mapped words, which will be represented by an index into + // the canonical_words and a Mapping; u16 when encoded. + let mut canonicalized_words = Vec::new(); + let mut unique_mapping = HashMap::new(); + + #[derive(Debug, PartialEq, Eq)] + enum UniqueMapping { + Canonical(usize), + Canonicalized(usize), + } + + // Map 0 first, so that it is the first canonical word. + // This is realistically not inefficient because 0 is not mapped to by + // anything else (a shift pattern could do it, but would be wasteful). + // + // However, 0s are quite common in the overall dataset, and it is quite + // wasteful to have to go through a mapping function to determine that + // we have a zero. + // + // FIXME: Experiment with choosing most common words in overall data set + // for canonical when possible. + while let Some((&to, _)) = mappings + .iter() + .find(|(&to, _)| to == 0) + .or_else(|| mappings.iter().max_by_key(|m| m.1.len())) + { + // Get the mapping with the most entries. Currently, no mapping can + // only exist transitively (i.e., there is no A, B, C such that A + // does not map to C and but A maps to B maps to C), so this is + // guaranteed to be acceptable. + // + // In the future, we may need a more sophisticated algorithm to + // identify which keys to prefer as canonical. + let mapped_from = mappings.remove(&to).unwrap(); + for (from, how) in &mapped_from { + // Remove the entries which mapped to this one. + // Noting that it should be associated with the Nth canonical word. + // + // We do not assert that this is present, because there may be + // no mappings to the `from` word; that's fine. + mappings.remove(from); + assert_eq!( + unique_mapping + .insert(*from, UniqueMapping::Canonicalized(canonicalized_words.len())), + None + ); + canonicalized_words.push((canonical_words.len(), *how)); + + // Remove the now-canonicalized word from other mappings, + // to ensure that we deprioritize them in the next iteration of + // the while loop. + for (_, mapped) in &mut mappings { + let mut i = 0; + while i != mapped.len() { + if mapped[i].0 == *from { + mapped.remove(i); + } else { + i += 1; + } + } + } + } + assert!( + unique_mapping + .insert(to, UniqueMapping::Canonical(canonical_words.len())) + .is_none() + ); + canonical_words.push(to); + + // Remove the now-canonical word from other mappings, to ensure that + // we deprioritize them in the next iteration of the while loop. + for (_, mapped) in &mut mappings { + let mut i = 0; + while i != mapped.len() { + if mapped[i].0 == to { + mapped.remove(i); + } else { + i += 1; + } + } + } + } + + // Any words which we couldn't shrink, just stick into the canonical + // words. + // + // FIXME: work harder -- there are more possibilities for mapping + // functions (e.g., multiplication, shifting instead of rotation, etc.) + // We'll probably always have some slack though so this loop will still + // be needed. + for &w in unique_words { + if !unique_mapping.contains_key(&w) { + assert!( + unique_mapping + .insert(w, UniqueMapping::Canonical(canonical_words.len())) + .is_none() + ); + canonical_words.push(w); + } + } + assert_eq!(canonicalized_words.len() + canonical_words.len(), unique_words.len()); + assert_eq!(unique_mapping.len(), unique_words.len()); + + let unique_mapping = unique_mapping + .into_iter() + .map(|(key, value)| { + ( + key, + match value { + UniqueMapping::Canonicalized(idx) => { + u8::try_from(canonical_words.len() + idx).unwrap() + } + UniqueMapping::Canonical(idx) => u8::try_from(idx).unwrap(), + }, + ) + }) + .collect::>(); + + let mut distinct_indices = BTreeSet::new(); + for &w in unique_words { + let idx = unique_mapping.get(&w).unwrap(); + assert!(distinct_indices.insert(idx)); + } + + const LOWER_6: u32 = (1 << 6) - 1; + + let canonicalized_words = canonicalized_words + .into_iter() + .map(|v| { + ( + u8::try_from(v.0).unwrap(), + match v.1 { + Mapping::RotateAndInvert(amount) => { + assert_eq!(amount, amount & LOWER_6); + 1 << 6 | (amount as u8) + } + Mapping::Rotate(amount) => { + assert_eq!(amount, amount & LOWER_6); + amount as u8 + } + Mapping::Invert => 1 << 6, + Mapping::ShiftRight(shift_by) => { + assert_eq!(shift_by, shift_by & LOWER_6); + 1 << 7 | (shift_by as u8) + } + }, + ) + }) + .collect::>(); + Canonicalized { unique_mapping, canonical_words, canonicalized_words } + } } diff --git a/src/tools/unicode-table-generator/src/skiplist.rs b/src/tools/unicode-table-generator/src/skiplist.rs new file mode 100644 index 00000000000..6e439968c3b --- /dev/null +++ b/src/tools/unicode-table-generator/src/skiplist.rs @@ -0,0 +1,98 @@ +use crate::fmt_list; +use crate::raw_emitter::RawEmitter; +use std::convert::TryInto; +use std::fmt::Write as _; +use std::ops::Range; + +/// This will get packed into a single u32 before inserting into the data set. +#[derive(Debug, PartialEq)] +struct ShortOffsetRunHeader { + /// Note, we only allow for 21 bits here. + prefix_sum: u32, + + /// Note, we actually only allow for 11 bits here. This should be enough -- + /// our largest sets are around ~1400 offsets long. + start_idx: u16, +} + +impl ShortOffsetRunHeader { + fn pack(&self) -> u32 { + assert!(self.start_idx < (1 << 11)); + assert!(self.prefix_sum < (1 << 21)); + + (self.start_idx as u32) << 21 | self.prefix_sum + } +} + +impl RawEmitter { + pub fn emit_skiplist(&mut self, ranges: &[Range]) { + let mut offsets = Vec::::new(); + let points = ranges.iter().flat_map(|r| vec![r.start, r.end]).collect::>(); + let mut offset = 0; + for pt in points { + let delta = pt - offset; + offsets.push(delta); + offset = pt; + } + // Guaranteed to terminate, as it's impossible to subtract a value this + // large from a valid char. + offsets.push(std::char::MAX as u32 + 1); + let mut coded_offsets: Vec = Vec::new(); + let mut short_offset_runs: Vec = vec![]; + let mut iter = offsets.iter().cloned(); + let mut prefix_sum = 0; + loop { + let mut any_elements = false; + let mut inserted = false; + let start = coded_offsets.len(); + for offset in iter.by_ref() { + any_elements = true; + prefix_sum += offset; + if let Ok(offset) = offset.try_into() { + coded_offsets.push(offset); + } else { + short_offset_runs.push(ShortOffsetRunHeader { + start_idx: start.try_into().unwrap(), + prefix_sum, + }); + // This is just needed to maintain indices even/odd + // correctly. + coded_offsets.push(0); + inserted = true; + break; + } + } + if !any_elements { + break; + } + // We always append the huge char::MAX offset to the end which + // should never be able to fit into the u8 offsets. + assert!(inserted); + } + + writeln!( + &mut self.file, + "static SHORT_OFFSET_RUNS: [u32; {}] = [{}];", + short_offset_runs.len(), + fmt_list(short_offset_runs.iter().map(|v| v.pack())) + ) + .unwrap(); + self.bytes_used += 4 * short_offset_runs.len(); + writeln!( + &mut self.file, + "static OFFSETS: [u8; {}] = [{}];", + coded_offsets.len(), + fmt_list(&coded_offsets) + ) + .unwrap(); + self.bytes_used += coded_offsets.len(); + + writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap(); + writeln!(&mut self.file, " super::skip_search(",).unwrap(); + writeln!(&mut self.file, " c as u32,").unwrap(); + writeln!(&mut self.file, " &SHORT_OFFSET_RUNS,").unwrap(); + writeln!(&mut self.file, " &OFFSETS,").unwrap(); + writeln!(&mut self.file, " )").unwrap(); + writeln!(&mut self.file, "}}").unwrap(); + } +} diff --git a/src/tools/unicode-table-generator/src/unicode_download.rs b/src/tools/unicode-table-generator/src/unicode_download.rs index 3f6de9ea3bb..fa57f650ac0 100644 --- a/src/tools/unicode-table-generator/src/unicode_download.rs +++ b/src/tools/unicode-table-generator/src/unicode_download.rs @@ -11,10 +11,15 @@ static RESOURCES: &[&str] = pub fn fetch_latest() { let directory = Path::new(UNICODE_DIRECTORY); + if directory.exists() { + eprintln!( + "Not refetching unicode data, already exists, please delete {:?} to regenerate", + directory + ); + return; + } if let Err(e) = std::fs::create_dir_all(directory) { - if e.kind() != std::io::ErrorKind::AlreadyExists { - panic!("Failed to create {:?}: {}", UNICODE_DIRECTORY, e); - } + panic!("Failed to create {:?}: {}", UNICODE_DIRECTORY, e); } let output = Command::new("curl").arg(URL_PREFIX.to_owned() + README).output().unwrap(); if !output.status.success() {