nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2025-04-28 19:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #115507 - cjgillot:relative-source-file, r=oli-obk

Browse Source 
Use relative positions inside a SourceFile.

This allows to remove the normalization of start positions for hashing, and simplify allocation of global address space.

cc `@Zoxc`
This commit is contained in:
bors 2023-09-05 21:03:56 +00:00
commit a991861ec9
20 changed files with 260 additions and 369 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
View File

@ -82,12 +82,9 @@ impl DebugContext {
match tcx.sess.source_map().lookup_line(span.lo()) { match tcx.sess.source_map().lookup_line(span.lo()) {
Ok(SourceFileAndLine { sf: file, line }) => { Ok(SourceFileAndLine { sf: file, line }) => {
let line_pos = file.lines(|lines| lines[line]); let line_pos = file.lines(|lines| lines[line]);
let col = file.relative_position(span.lo()) - line_pos;
( (file, u64::try_from(line).unwrap() + 1, u64::from(col.to_u32()) + 1)
file,
u64::try_from(line).unwrap() + 1,
u64::from((span.lo() - line_pos).to_u32()) + 1,
)
} }
Err(file) => (file, 0, 0), Err(file) => (file, 0, 0),
} }

4
compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
View File

@ -68,7 +68,7 @@ fn make_mir_scope<'ll, 'tcx>(
let file = cx.sess().source_map().lookup_source_file(mir.span.lo()); let file = cx.sess().source_map().lookup_source_file(mir.span.lo());
debug_context.scopes[scope] = DebugScope { debug_context.scopes[scope] = DebugScope {
file_start_pos: file.start_pos, file_start_pos: file.start_pos,
file_end_pos: file.end_pos, file_end_pos: file.end_position(),
..debug_context.scopes[scope] ..debug_context.scopes[scope]
}; };
instantiated.insert(scope); instantiated.insert(scope);
@ -120,7 +120,7 @@ fn make_mir_scope<'ll, 'tcx>(
dbg_scope, dbg_scope,
inlined_at: inlined_at.or(parent_scope.inlined_at), inlined_at: inlined_at.or(parent_scope.inlined_at),
file_start_pos: loc.file.start_pos, file_start_pos: loc.file.start_pos,
file_end_pos: loc.file.end_pos, file_end_pos: loc.file.end_position(),
}; };
instantiated.insert(scope); instantiated.insert(scope);
} }

2
compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
View File

@ -267,7 +267,7 @@ impl CodegenCx<'_, '_> {
// Use 1-based indexing. // Use 1-based indexing.
let line = (line + 1) as u32; let line = (line + 1) as u32;
let col = (pos - line_pos).to_u32() + 1; let col = (file.relative_position(pos) - line_pos).to_u32() + 1;
(file, line, col) (file, line, col)
} }

24
compiler/rustc_metadata/src/rmeta/decoder.rs
View File

@ -1501,11 +1501,12 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
// We can't reuse an existing SourceFile, so allocate a new one // We can't reuse an existing SourceFile, so allocate a new one
// containing the information we need. // containing the information we need.
let original_end_pos = source_file_to_import.end_position();
let rustc_span::SourceFile { let rustc_span::SourceFile {
mut name, mut name,
src_hash, src_hash,
start_pos, start_pos: original_start_pos,
end_pos, source_len,
lines, lines,
multibyte_chars, multibyte_chars,
non_narrow_chars, non_narrow_chars,
@ -1547,35 +1548,32 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
// on `try_to_translate_virtual_to_real`). // on `try_to_translate_virtual_to_real`).
try_to_translate_virtual_to_real(&mut name); try_to_translate_virtual_to_real(&mut name);
let source_length = (end_pos - start_pos).to_usize();
let local_version = sess.source_map().new_imported_source_file( let local_version = sess.source_map().new_imported_source_file(
name, name,
src_hash, src_hash,
name_hash, name_hash,
source_length, source_len.to_u32(),
self.cnum, self.cnum,
lines, lines,
multibyte_chars, multibyte_chars,
non_narrow_chars, non_narrow_chars,
normalized_pos, normalized_pos,
start_pos,
source_file_index, source_file_index,
); );
debug!( debug!(
"CrateMetaData::imported_source_files alloc \ "CrateMetaData::imported_source_files alloc \
source_file {:?} original (start_pos {:?} end_pos {:?}) \ source_file {:?} original (start_pos {:?} source_len {:?}) \
translated (start_pos {:?} end_pos {:?})", translated (start_pos {:?} source_len {:?})",
local_version.name, local_version.name,
start_pos, original_start_pos,
end_pos, source_len,
local_version.start_pos, local_version.start_pos,
local_version.end_pos local_version.source_len
); );
ImportedSourceFile { ImportedSourceFile {
original_start_pos: start_pos, original_start_pos,
original_end_pos: end_pos, original_end_pos,
translated_source_file: local_version, translated_source_file: local_version,
} }
}) })

5
compiler/rustc_middle/src/query/on_disk_cache.rs
View File

@ -22,7 +22,7 @@ use rustc_span::hygiene::{
ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData, ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData,
}; };
use rustc_span::source_map::{SourceMap, StableSourceFileId}; use rustc_span::source_map::{SourceMap, StableSourceFileId};
use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span}; use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, RelativeBytePos, SourceFile, Span};
use rustc_span::{CachingSourceMapView, Symbol}; use rustc_span::{CachingSourceMapView, Symbol};
use std::collections::hash_map::Entry; use std::collections::hash_map::Entry;
use std::io; use std::io;
@ -688,11 +688,12 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span {
let file_lo_index = SourceFileIndex::decode(decoder); let file_lo_index = SourceFileIndex::decode(decoder);
let line_lo = usize::decode(decoder); let line_lo = usize::decode(decoder);
let col_lo = BytePos::decode(decoder); let col_lo = RelativeBytePos::decode(decoder);
let len = BytePos::decode(decoder); let len = BytePos::decode(decoder);
let file_lo = decoder.file_index_to_file(file_lo_index); let file_lo = decoder.file_index_to_file(file_lo_index);
let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo); let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo);
let lo = file_lo.absolute_position(lo);
let hi = lo + len; let hi = lo + len;
Span::new(lo, hi, ctxt, parent) Span::new(lo, hi, ctxt, parent)

33
compiler/rustc_mir_transform/src/coverage/mod.rs
View File

@ -28,7 +28,7 @@ use rustc_middle::mir::{
use rustc_middle::ty::TyCtxt; use rustc_middle::ty::TyCtxt;
use rustc_span::def_id::DefId; use rustc_span::def_id::DefId;
use rustc_span::source_map::SourceMap; use rustc_span::source_map::SourceMap;
use rustc_span::{CharPos, ExpnKind, Pos, SourceFile, Span, Symbol}; use rustc_span::{ExpnKind, SourceFile, Span, Symbol};
/// A simple error message wrapper for `coverage::Error`s. /// A simple error message wrapper for `coverage::Error`s.
#[derive(Debug)] #[derive(Debug)]
@ -314,8 +314,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
}; };
graphviz_data.add_bcb_coverage_span_with_counter(bcb, &covspan, &counter_kind); graphviz_data.add_bcb_coverage_span_with_counter(bcb, &covspan, &counter_kind);
let code_region = let code_region = make_code_region(source_map, file_name, span, body_span);
make_code_region(source_map, file_name, &self.source_file, span, body_span);
inject_statement( inject_statement(
self.mir_body, self.mir_body,
@ -510,40 +509,36 @@ fn inject_intermediate_expression(mir_body: &mut mir::Body<'_>, expression: Cove
fn make_code_region( fn make_code_region(
source_map: &SourceMap, source_map: &SourceMap,
file_name: Symbol, file_name: Symbol,
source_file: &Lrc<SourceFile>,
span: Span, span: Span,
body_span: Span, body_span: Span,
) -> CodeRegion { ) -> CodeRegion {
debug!( debug!(
"Called make_code_region(file_name={}, source_file={:?}, span={}, body_span={})", "Called make_code_region(file_name={}, span={}, body_span={})",
file_name, file_name,
source_file,
source_map.span_to_diagnostic_string(span), source_map.span_to_diagnostic_string(span),
source_map.span_to_diagnostic_string(body_span) source_map.span_to_diagnostic_string(body_span)
); );
let (start_line, mut start_col) = source_file.lookup_file_pos(span.lo()); let (file, mut start_line, mut start_col, mut end_line, mut end_col) =
let (end_line, end_col) = if span.hi() == span.lo() { source_map.span_to_location_info(span);
let (end_line, mut end_col) = (start_line, start_col); if span.hi() == span.lo() {
// Extend an empty span by one character so the region will be counted. // Extend an empty span by one character so the region will be counted.
let CharPos(char_pos) = start_col;
if span.hi() == body_span.hi() { if span.hi() == body_span.hi() {
start_col = CharPos(char_pos.saturating_sub(1)); start_col = start_col.saturating_sub(1);
} else { } else {
end_col = CharPos(char_pos + 1); end_col = start_col + 1;
} }
(end_line, end_col)
} else {
source_file.lookup_file_pos(span.hi())
}; };
let start_line = source_map.doctest_offset_line(&source_file.name, start_line); if let Some(file) = file {
let end_line = source_map.doctest_offset_line(&source_file.name, end_line); start_line = source_map.doctest_offset_line(&file.name, start_line);
end_line = source_map.doctest_offset_line(&file.name, end_line);
}
CodeRegion { CodeRegion {
file_name, file_name,
start_line: start_line as u32, start_line: start_line as u32,
start_col: start_col.to_u32() + 1, start_col: start_col as u32,
end_line: end_line as u32, end_line: end_line as u32,
end_col: end_col.to_u32() + 1, end_col: end_col as u32,
} }
} }

2
compiler/rustc_parse/src/lib.rs
View File

@ -132,7 +132,7 @@ fn maybe_source_file_to_parser(
sess: &ParseSess, sess: &ParseSess,
source_file: Lrc<SourceFile>, source_file: Lrc<SourceFile>,
) -> Result<Parser<'_>, Vec<Diagnostic>> { ) -> Result<Parser<'_>, Vec<Diagnostic>> {
let end_pos = source_file.end_pos; let end_pos = source_file.end_position();
let stream = maybe_file_to_stream(sess, source_file, None)?; let stream = maybe_file_to_stream(sess, source_file, None)?;
let mut parser = stream_to_parser(sess, stream, None); let mut parser = stream_to_parser(sess, stream, None);
if parser.token == token::Eof { if parser.token == token::Eof {

40
compiler/rustc_query_system/src/ich/impls_syntax.rs
View File

@ -5,7 +5,7 @@ use crate::ich::StableHashingContext;
use rustc_ast as ast; use rustc_ast as ast;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_span::{BytePos, NormalizedPos, SourceFile}; use rustc_span::SourceFile;
use std::assert_matches::assert_matches; use std::assert_matches::assert_matches;
use smallvec::SmallVec; use smallvec::SmallVec;
@ -67,8 +67,8 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
src: _, src: _,
ref src_hash, ref src_hash,
external_src: _, external_src: _,
start_pos, start_pos: _,
end_pos: _, source_len: _,
lines: _, lines: _,
ref multibyte_chars, ref multibyte_chars,
ref non_narrow_chars, ref non_narrow_chars,
@ -85,56 +85,30 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
// We only hash the relative position within this source_file // We only hash the relative position within this source_file
lines.len().hash_stable(hcx, hasher); lines.len().hash_stable(hcx, hasher);
for &line in lines.iter() { for &line in lines.iter() {
stable_byte_pos(line, start_pos).hash_stable(hcx, hasher); line.hash_stable(hcx, hasher);
} }
}); });
// We only hash the relative position within this source_file // We only hash the relative position within this source_file
multibyte_chars.len().hash_stable(hcx, hasher); multibyte_chars.len().hash_stable(hcx, hasher);
for &char_pos in multibyte_chars.iter() { for &char_pos in multibyte_chars.iter() {
stable_multibyte_char(char_pos, start_pos).hash_stable(hcx, hasher); char_pos.hash_stable(hcx, hasher);
} }
non_narrow_chars.len().hash_stable(hcx, hasher); non_narrow_chars.len().hash_stable(hcx, hasher);
for &char_pos in non_narrow_chars.iter() { for &char_pos in non_narrow_chars.iter() {
stable_non_narrow_char(char_pos, start_pos).hash_stable(hcx, hasher); char_pos.hash_stable(hcx, hasher);
} }
normalized_pos.len().hash_stable(hcx, hasher); normalized_pos.len().hash_stable(hcx, hasher);
for &char_pos in normalized_pos.iter() { for &char_pos in normalized_pos.iter() {
stable_normalized_pos(char_pos, start_pos).hash_stable(hcx, hasher); char_pos.hash_stable(hcx, hasher);
} }
cnum.hash_stable(hcx, hasher); cnum.hash_stable(hcx, hasher);
} }
} }
fn stable_byte_pos(pos: BytePos, source_file_start: BytePos) -> u32 {
pos.0 - source_file_start.0
}
fn stable_multibyte_char(mbc: rustc_span::MultiByteChar, source_file_start: BytePos) -> (u32, u32) {
let rustc_span::MultiByteChar { pos, bytes } = mbc;
(pos.0 - source_file_start.0, bytes as u32)
}
fn stable_non_narrow_char(
swc: rustc_span::NonNarrowChar,
source_file_start: BytePos,
) -> (u32, u32) {
let pos = swc.pos();
let width = swc.width();
(pos.0 - source_file_start.0, width as u32)
}
fn stable_normalized_pos(np: NormalizedPos, source_file_start: BytePos) -> (u32, u32) {
let NormalizedPos { pos, diff } = np;
(pos.0 - source_file_start.0, diff)
}
impl<'tcx> HashStable<StableHashingContext<'tcx>> for rustc_feature::Features { impl<'tcx> HashStable<StableHashingContext<'tcx>> for rustc_feature::Features {
fn hash_stable(&self, hcx: &mut StableHashingContext<'tcx>, hasher: &mut StableHasher) { fn hash_stable(&self, hcx: &mut StableHashingContext<'tcx>, hasher: &mut StableHasher) {
// Unfortunately we cannot exhaustively list fields here, since the // Unfortunately we cannot exhaustively list fields here, since the

48
compiler/rustc_span/src/analyze_source_file.rs
View File

@ -11,26 +11,19 @@ mod tests;
/// is detected at runtime. /// is detected at runtime.
pub fn analyze_source_file( pub fn analyze_source_file(
src: &str, src: &str,
source_file_start_pos: BytePos, ) -> (Vec<RelativeBytePos>, Vec<MultiByteChar>, Vec<NonNarrowChar>) {
) -> (Vec<BytePos>, Vec<MultiByteChar>, Vec<NonNarrowChar>) { let mut lines = vec![RelativeBytePos::from_u32(0)];
let mut lines = vec![source_file_start_pos];
let mut multi_byte_chars = vec![]; let mut multi_byte_chars = vec![];
let mut non_narrow_chars = vec![]; let mut non_narrow_chars = vec![];
// Calls the right implementation, depending on hardware support available. // Calls the right implementation, depending on hardware support available.
analyze_source_file_dispatch( analyze_source_file_dispatch(src, &mut lines, &mut multi_byte_chars, &mut non_narrow_chars);
src,
source_file_start_pos,
&mut lines,
&mut multi_byte_chars,
&mut non_narrow_chars,
);
// The code above optimistically registers a new line *after* each \n // The code above optimistically registers a new line *after* each \n
// it encounters. If that point is already outside the source_file, remove // it encounters. If that point is already outside the source_file, remove
// it again. // it again.
if let Some(&last_line_start) = lines.last() { if let Some(&last_line_start) = lines.last() {
let source_file_end = source_file_start_pos + BytePos::from_usize(src.len()); let source_file_end = RelativeBytePos::from_usize(src.len());
assert!(source_file_end >= last_line_start); assert!(source_file_end >= last_line_start);
if last_line_start == source_file_end { if last_line_start == source_file_end {
lines.pop(); lines.pop();
@ -43,14 +36,12 @@ pub fn analyze_source_file(
cfg_if::cfg_if! { cfg_if::cfg_if! {
if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
fn analyze_source_file_dispatch(src: &str, fn analyze_source_file_dispatch(src: &str,
source_file_start_pos: BytePos, lines: &mut Vec<RelativeBytePos>,
lines: &mut Vec<BytePos>,
multi_byte_chars: &mut Vec<MultiByteChar>, multi_byte_chars: &mut Vec<MultiByteChar>,
non_narrow_chars: &mut Vec<NonNarrowChar>) { non_narrow_chars: &mut Vec<NonNarrowChar>) {
if is_x86_feature_detected!("sse2") { if is_x86_feature_detected!("sse2") {
unsafe { unsafe {
analyze_source_file_sse2(src, analyze_source_file_sse2(src,
source_file_start_pos,
lines, lines,
multi_byte_chars, multi_byte_chars,
non_narrow_chars); non_narrow_chars);
@ -58,7 +49,7 @@ cfg_if::cfg_if! {
} else { } else {
analyze_source_file_generic(src, analyze_source_file_generic(src,
src.len(), src.len(),
source_file_start_pos, RelativeBytePos::from_u32(0),
lines, lines,
multi_byte_chars, multi_byte_chars,
non_narrow_chars); non_narrow_chars);
@ -72,8 +63,7 @@ cfg_if::cfg_if! {
/// SSE2 intrinsics to quickly find all newlines. /// SSE2 intrinsics to quickly find all newlines.
#[target_feature(enable = "sse2")] #[target_feature(enable = "sse2")]
unsafe fn analyze_source_file_sse2(src: &str, unsafe fn analyze_source_file_sse2(src: &str,
output_offset: BytePos, lines: &mut Vec<RelativeBytePos>,
lines: &mut Vec<BytePos>,
multi_byte_chars: &mut Vec<MultiByteChar>, multi_byte_chars: &mut Vec<MultiByteChar>,
non_narrow_chars: &mut Vec<NonNarrowChar>) { non_narrow_chars: &mut Vec<NonNarrowChar>) {
#[cfg(target_arch = "x86")] #[cfg(target_arch = "x86")]
@ -129,8 +119,7 @@ cfg_if::cfg_if! {
if control_char_mask == newlines_mask { if control_char_mask == newlines_mask {
// All control characters are newlines, record them // All control characters are newlines, record them
let mut newlines_mask = 0xFFFF0000 | newlines_mask as u32; let mut newlines_mask = 0xFFFF0000 | newlines_mask as u32;
let output_offset = output_offset + let output_offset = RelativeBytePos::from_usize(chunk_index * CHUNK_SIZE + 1);
BytePos::from_usize(chunk_index * CHUNK_SIZE + 1);
loop { loop {
let index = newlines_mask.trailing_zeros(); let index = newlines_mask.trailing_zeros();
@ -140,7 +129,7 @@ cfg_if::cfg_if! {
break break
} }
lines.push(BytePos(index) + output_offset); lines.push(RelativeBytePos(index) + output_offset);
// Clear the bit, so we can find the next one. // Clear the bit, so we can find the next one.
newlines_mask &= (!1) << index; newlines_mask &= (!1) << index;
@ -165,7 +154,7 @@ cfg_if::cfg_if! {
intra_chunk_offset = analyze_source_file_generic( intra_chunk_offset = analyze_source_file_generic(
&src[scan_start .. ], &src[scan_start .. ],
CHUNK_SIZE - intra_chunk_offset, CHUNK_SIZE - intra_chunk_offset,
BytePos::from_usize(scan_start) + output_offset, RelativeBytePos::from_usize(scan_start),
lines, lines,
multi_byte_chars, multi_byte_chars,
non_narrow_chars non_narrow_chars
@ -177,7 +166,7 @@ cfg_if::cfg_if! {
if tail_start < src.len() { if tail_start < src.len() {
analyze_source_file_generic(&src[tail_start ..], analyze_source_file_generic(&src[tail_start ..],
src.len() - tail_start, src.len() - tail_start,
output_offset + BytePos::from_usize(tail_start), RelativeBytePos::from_usize(tail_start),
lines, lines,
multi_byte_chars, multi_byte_chars,
non_narrow_chars); non_narrow_chars);
@ -187,13 +176,12 @@ cfg_if::cfg_if! {
// The target (or compiler version) does not support SSE2 ... // The target (or compiler version) does not support SSE2 ...
fn analyze_source_file_dispatch(src: &str, fn analyze_source_file_dispatch(src: &str,
source_file_start_pos: BytePos, lines: &mut Vec<RelativeBytePos>,
lines: &mut Vec<BytePos>,
multi_byte_chars: &mut Vec<MultiByteChar>, multi_byte_chars: &mut Vec<MultiByteChar>,
non_narrow_chars: &mut Vec<NonNarrowChar>) { non_narrow_chars: &mut Vec<NonNarrowChar>) {
analyze_source_file_generic(src, analyze_source_file_generic(src,
src.len(), src.len(),
source_file_start_pos, RelativeBytePos::from_u32(0),
lines, lines,
multi_byte_chars, multi_byte_chars,
non_narrow_chars); non_narrow_chars);
@ -207,8 +195,8 @@ cfg_if::cfg_if! {
fn analyze_source_file_generic( fn analyze_source_file_generic(
src: &str, src: &str,
scan_len: usize, scan_len: usize,
output_offset: BytePos, output_offset: RelativeBytePos,
lines: &mut Vec<BytePos>, lines: &mut Vec<RelativeBytePos>,
multi_byte_chars: &mut Vec<MultiByteChar>, multi_byte_chars: &mut Vec<MultiByteChar>,
non_narrow_chars: &mut Vec<NonNarrowChar>, non_narrow_chars: &mut Vec<NonNarrowChar>,
) -> usize { ) -> usize {
@ -230,11 +218,11 @@ fn analyze_source_file_generic(
// This is an ASCII control character, it could be one of the cases // This is an ASCII control character, it could be one of the cases
// that are interesting to us. // that are interesting to us.
let pos = BytePos::from_usize(i) + output_offset; let pos = RelativeBytePos::from_usize(i) + output_offset;
match byte { match byte {
b'\n' => { b'\n' => {
lines.push(pos + BytePos(1)); lines.push(pos + RelativeBytePos(1));
} }
b'\t' => { b'\t' => {
non_narrow_chars.push(NonNarrowChar::Tab(pos)); non_narrow_chars.push(NonNarrowChar::Tab(pos));
@ -250,7 +238,7 @@ fn analyze_source_file_generic(
let c = src[i..].chars().next().unwrap(); let c = src[i..].chars().next().unwrap();
char_len = c.len_utf8(); char_len = c.len_utf8();
let pos = BytePos::from_usize(i) + output_offset; let pos = RelativeBytePos::from_usize(i) + output_offset;
if char_len > 1 { if char_len > 1 {
assert!((2..=4).contains(&char_len)); assert!((2..=4).contains(&char_len));

29
compiler/rustc_span/src/analyze_source_file/tests.rs
View File

@ -3,29 +3,28 @@ use super::*;
macro_rules! test { macro_rules! test {
(case: $test_name:ident, (case: $test_name:ident,
text: $text:expr, text: $text:expr,
source_file_start_pos: $source_file_start_pos:expr,
lines: $lines:expr, lines: $lines:expr,
multi_byte_chars: $multi_byte_chars:expr, multi_byte_chars: $multi_byte_chars:expr,
non_narrow_chars: $non_narrow_chars:expr,) => { non_narrow_chars: $non_narrow_chars:expr,) => {
#[test] #[test]
fn $test_name() { fn $test_name() {
let (lines, multi_byte_chars, non_narrow_chars) = let (lines, multi_byte_chars, non_narrow_chars) = analyze_source_file($text);
analyze_source_file($text, BytePos($source_file_start_pos));
let expected_lines: Vec<BytePos> = $lines.into_iter().map(BytePos).collect(); let expected_lines: Vec<RelativeBytePos> =
$lines.into_iter().map(RelativeBytePos).collect();
assert_eq!(lines, expected_lines); assert_eq!(lines, expected_lines);
let expected_mbcs: Vec<MultiByteChar> = $multi_byte_chars let expected_mbcs: Vec<MultiByteChar> = $multi_byte_chars
.into_iter() .into_iter()
.map(|(pos, bytes)| MultiByteChar { pos: BytePos(pos), bytes }) .map(|(pos, bytes)| MultiByteChar { pos: RelativeBytePos(pos), bytes })
.collect(); .collect();
assert_eq!(multi_byte_chars, expected_mbcs); assert_eq!(multi_byte_chars, expected_mbcs);
let expected_nncs: Vec<NonNarrowChar> = $non_narrow_chars let expected_nncs: Vec<NonNarrowChar> = $non_narrow_chars
.into_iter() .into_iter()
.map(|(pos, width)| NonNarrowChar::new(BytePos(pos), width)) .map(|(pos, width)| NonNarrowChar::new(RelativeBytePos(pos), width))
.collect(); .collect();
assert_eq!(non_narrow_chars, expected_nncs); assert_eq!(non_narrow_chars, expected_nncs);
@ -36,7 +35,6 @@ macro_rules! test {
test!( test!(
case: empty_text, case: empty_text,
text: "", text: "",
source_file_start_pos: 0,
lines: vec![], lines: vec![],
multi_byte_chars: vec![], multi_byte_chars: vec![],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -45,7 +43,6 @@ test!(
test!( test!(
case: newlines_short, case: newlines_short,
text: "a\nc", text: "a\nc",
source_file_start_pos: 0,
lines: vec![0, 2], lines: vec![0, 2],
multi_byte_chars: vec![], multi_byte_chars: vec![],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -54,7 +51,6 @@ test!(
test!( test!(
case: newlines_long, case: newlines_long,
text: "012345678\nabcdef012345678\na", text: "012345678\nabcdef012345678\na",
source_file_start_pos: 0,
lines: vec![0, 10, 26], lines: vec![0, 10, 26],
multi_byte_chars: vec![], multi_byte_chars: vec![],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -63,7 +59,6 @@ test!(
test!( test!(
case: newline_and_multi_byte_char_in_same_chunk, case: newline_and_multi_byte_char_in_same_chunk,
text: "01234β789\nbcdef0123456789abcdef", text: "01234β789\nbcdef0123456789abcdef",
source_file_start_pos: 0,
lines: vec![0, 11], lines: vec![0, 11],
multi_byte_chars: vec![(5, 2)], multi_byte_chars: vec![(5, 2)],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -72,7 +67,6 @@ test!(
test!( test!(
case: newline_and_control_char_in_same_chunk, case: newline_and_control_char_in_same_chunk,
text: "01234\u{07}6789\nbcdef0123456789abcdef", text: "01234\u{07}6789\nbcdef0123456789abcdef",
source_file_start_pos: 0,
lines: vec![0, 11], lines: vec![0, 11],
multi_byte_chars: vec![], multi_byte_chars: vec![],
non_narrow_chars: vec![(5, 0)], non_narrow_chars: vec![(5, 0)],
@ -81,7 +75,6 @@ test!(
test!( test!(
case: multi_byte_char_short, case: multi_byte_char_short,
text: "aβc", text: "aβc",
source_file_start_pos: 0,
lines: vec![0], lines: vec![0],
multi_byte_chars: vec![(1, 2)], multi_byte_chars: vec![(1, 2)],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -90,7 +83,6 @@ test!(
test!( test!(
case: multi_byte_char_long, case: multi_byte_char_long,
text: "0123456789abcΔf012345β", text: "0123456789abcΔf012345β",
source_file_start_pos: 0,
lines: vec![0], lines: vec![0],
multi_byte_chars: vec![(13, 2), (22, 2)], multi_byte_chars: vec![(13, 2), (22, 2)],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -99,7 +91,6 @@ test!(
test!( test!(
case: multi_byte_char_across_chunk_boundary, case: multi_byte_char_across_chunk_boundary,
text: "0123456789abcdeΔ123456789abcdef01234", text: "0123456789abcdeΔ123456789abcdef01234",
source_file_start_pos: 0,
lines: vec![0], lines: vec![0],
multi_byte_chars: vec![(15, 2)], multi_byte_chars: vec![(15, 2)],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -108,7 +99,6 @@ test!(
test!( test!(
case: multi_byte_char_across_chunk_boundary_tail, case: multi_byte_char_across_chunk_boundary_tail,
text: "0123456789abcdeΔ....", text: "0123456789abcdeΔ....",
source_file_start_pos: 0,
lines: vec![0], lines: vec![0],
multi_byte_chars: vec![(15, 2)], multi_byte_chars: vec![(15, 2)],
non_narrow_chars: vec![], non_narrow_chars: vec![],
@ -117,7 +107,6 @@ test!(
test!( test!(
case: non_narrow_short, case: non_narrow_short,
text: "0\t2", text: "0\t2",
source_file_start_pos: 0,
lines: vec![0], lines: vec![0],
multi_byte_chars: vec![], multi_byte_chars: vec![],
non_narrow_chars: vec![(1, 4)], non_narrow_chars: vec![(1, 4)],
@ -126,7 +115,6 @@ test!(
test!( test!(
case: non_narrow_long, case: non_narrow_long,
text: "01\t3456789abcdef01234567\u{07}9", text: "01\t3456789abcdef01234567\u{07}9",
source_file_start_pos: 0,
lines: vec![0], lines: vec![0],
multi_byte_chars: vec![], multi_byte_chars: vec![],
non_narrow_chars: vec![(2, 4), (24, 0)], non_narrow_chars: vec![(2, 4), (24, 0)],
@ -135,8 +123,7 @@ test!(
test!( test!(
case: output_offset_all, case: output_offset_all,
text: "01\t345\n789abcΔf01234567\u{07}9\nbcΔf", text: "01\t345\n789abcΔf01234567\u{07}9\nbcΔf",
source_file_start_pos: 1000, lines: vec![0, 7, 27],
lines: vec![0 + 1000, 7 + 1000, 27 + 1000], multi_byte_chars: vec![(13, 2), (29, 2)],
multi_byte_chars: vec![(13 + 1000, 2), (29 + 1000, 2)], non_narrow_chars: vec![(2, 4), (24, 0)],
non_narrow_chars: vec![(2 + 1000, 4), (24 + 1000, 0)],
); );

15
compiler/rustc_span/src/caching_source_map_view.rs
View File

@ -1,5 +1,5 @@
use crate::source_map::SourceMap; use crate::source_map::SourceMap;
use crate::{BytePos, SourceFile, SpanData}; use crate::{BytePos, Pos, RelativeBytePos, SourceFile, SpanData};
use rustc_data_structures::sync::Lrc; use rustc_data_structures::sync::Lrc;
use std::ops::Range; use std::ops::Range;
@ -37,6 +37,7 @@ impl CacheEntry {
self.file_index = file_idx; self.file_index = file_idx;
} }
let pos = self.file.relative_position(pos);
let line_index = self.file.lookup_line(pos).unwrap(); let line_index = self.file.lookup_line(pos).unwrap();
let line_bounds = self.file.line_bounds(line_index); let line_bounds = self.file.line_bounds(line_index);
self.line_number = line_index + 1; self.line_number = line_index + 1;
@ -79,7 +80,7 @@ impl<'sm> CachingSourceMapView<'sm> {
pub fn byte_pos_to_line_and_col( pub fn byte_pos_to_line_and_col(
&mut self, &mut self,
pos: BytePos, pos: BytePos,
) -> Option<(Lrc<SourceFile>, usize, BytePos)> { ) -> Option<(Lrc<SourceFile>, usize, RelativeBytePos)> {
self.time_stamp += 1; self.time_stamp += 1;
// Check if the position is in one of the cached lines // Check if the position is in one of the cached lines
@ -88,11 +89,8 @@ impl<'sm> CachingSourceMapView<'sm> {
let cache_entry = &mut self.line_cache[cache_idx as usize]; let cache_entry = &mut self.line_cache[cache_idx as usize];
cache_entry.touch(self.time_stamp); cache_entry.touch(self.time_stamp);
return Some(( let col = RelativeBytePos(pos.to_u32() - cache_entry.line.start.to_u32());
cache_entry.file.clone(), return Some((cache_entry.file.clone(), cache_entry.line_number, col));
cache_entry.line_number,
pos - cache_entry.line.start,
));
} }
// No cache hit ... // No cache hit ...
@ -108,7 +106,8 @@ impl<'sm> CachingSourceMapView<'sm> {
let cache_entry = &mut self.line_cache[oldest]; let cache_entry = &mut self.line_cache[oldest];
cache_entry.update(new_file_and_idx, pos, self.time_stamp); cache_entry.update(new_file_and_idx, pos, self.time_stamp);
Some((cache_entry.file.clone(), cache_entry.line_number, pos - cache_entry.line.start)) let col = RelativeBytePos(pos.to_u32() - cache_entry.line.start.to_u32());
Some((cache_entry.file.clone(), cache_entry.line_number, col))
} }
pub fn span_data_to_lines_and_cols( pub fn span_data_to_lines_and_cols(

195
compiler/rustc_span/src/lib.rs
View File

@ -1107,27 +1107,27 @@ impl fmt::Debug for SpanData {
} }
/// Identifies an offset of a multi-byte character in a `SourceFile`. /// Identifies an offset of a multi-byte character in a `SourceFile`.
#[derive(Copy, Clone, Encodable, Decodable, Eq, PartialEq, Debug)] #[derive(Copy, Clone, Encodable, Decodable, Eq, PartialEq, Debug, HashStable_Generic)]
pub struct MultiByteChar { pub struct MultiByteChar {
/// The absolute offset of the character in the `SourceMap`. /// The relative offset of the character in the `SourceFile`.
pub pos: BytePos, pub pos: RelativeBytePos,
/// The number of bytes, `>= 2`. /// The number of bytes, `>= 2`.
pub bytes: u8, pub bytes: u8,
} }
/// Identifies an offset of a non-narrow character in a `SourceFile`. /// Identifies an offset of a non-narrow character in a `SourceFile`.
#[derive(Copy, Clone, Encodable, Decodable, Eq, PartialEq, Debug)] #[derive(Copy, Clone, Encodable, Decodable, Eq, PartialEq, Debug, HashStable_Generic)]
pub enum NonNarrowChar { pub enum NonNarrowChar {
/// Represents a zero-width character. /// Represents a zero-width character.
ZeroWidth(BytePos), ZeroWidth(RelativeBytePos),
/// Represents a wide (full-width) character. /// Represents a wide (full-width) character.
Wide(BytePos), Wide(RelativeBytePos),
/// Represents a tab character, represented visually with a width of 4 characters. /// Represents a tab character, represented visually with a width of 4 characters.
Tab(BytePos), Tab(RelativeBytePos),
} }
impl NonNarrowChar { impl NonNarrowChar {
fn new(pos: BytePos, width: usize) -> Self { fn new(pos: RelativeBytePos, width: usize) -> Self {
match width { match width {
0 => NonNarrowChar::ZeroWidth(pos), 0 => NonNarrowChar::ZeroWidth(pos),
2 => NonNarrowChar::Wide(pos), 2 => NonNarrowChar::Wide(pos),
@ -1136,8 +1136,8 @@ impl NonNarrowChar {
} }
} }
/// Returns the absolute offset of the character in the `SourceMap`. /// Returns the relative offset of the character in the `SourceFile`.
pub fn pos(&self) -> BytePos { pub fn pos(&self) -> RelativeBytePos {
match *self { match *self {
NonNarrowChar::ZeroWidth(p) | NonNarrowChar::Wide(p) | NonNarrowChar::Tab(p) => p, NonNarrowChar::ZeroWidth(p) | NonNarrowChar::Wide(p) | NonNarrowChar::Tab(p) => p,
} }
@ -1153,10 +1153,10 @@ impl NonNarrowChar {
} }
} }
impl Add<BytePos> for NonNarrowChar { impl Add<RelativeBytePos> for NonNarrowChar {
type Output = Self; type Output = Self;
fn add(self, rhs: BytePos) -> Self { fn add(self, rhs: RelativeBytePos) -> Self {
match self { match self {
NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos + rhs), NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos + rhs),
NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos + rhs), NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos + rhs),
@ -1165,10 +1165,10 @@ impl Add<BytePos> for NonNarrowChar {
} }
} }
impl Sub<BytePos> for NonNarrowChar { impl Sub<RelativeBytePos> for NonNarrowChar {
type Output = Self; type Output = Self;
fn sub(self, rhs: BytePos) -> Self { fn sub(self, rhs: RelativeBytePos) -> Self {
match self { match self {
NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos - rhs), NonNarrowChar::ZeroWidth(pos) => NonNarrowChar::ZeroWidth(pos - rhs),
NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos - rhs), NonNarrowChar::Wide(pos) => NonNarrowChar::Wide(pos - rhs),
@ -1178,10 +1178,10 @@ impl Sub<BytePos> for NonNarrowChar {
} }
/// Identifies an offset of a character that was normalized away from `SourceFile`. /// Identifies an offset of a character that was normalized away from `SourceFile`.
#[derive(Copy, Clone, Encodable, Decodable, Eq, PartialEq, Debug)] #[derive(Copy, Clone, Encodable, Decodable, Eq, PartialEq, Debug, HashStable_Generic)]
pub struct NormalizedPos { pub struct NormalizedPos {
/// The absolute offset of the character in the `SourceMap`. /// The relative offset of the character in the `SourceFile`.
pub pos: BytePos, pub pos: RelativeBytePos,
/// The difference between original and normalized string at position. /// The difference between original and normalized string at position.
pub diff: u32, pub diff: u32,
} }
@ -1293,7 +1293,7 @@ impl SourceFileHash {
#[derive(Clone)] #[derive(Clone)]
pub enum SourceFileLines { pub enum SourceFileLines {
/// The source file lines, in decoded (random-access) form. /// The source file lines, in decoded (random-access) form.
Lines(Vec<BytePos>), Lines(Vec<RelativeBytePos>),
/// The source file lines, in undecoded difference list form. /// The source file lines, in undecoded difference list form.
Diffs(SourceFileDiffs), Diffs(SourceFileDiffs),
@ -1314,11 +1314,6 @@ impl SourceFileLines {
/// small crates where very little of `std`'s metadata is used. /// small crates where very little of `std`'s metadata is used.
#[derive(Clone)] #[derive(Clone)]
pub struct SourceFileDiffs { pub struct SourceFileDiffs {
/// Position of the first line. Note that this is always encoded as a
/// `BytePos` because it is often much larger than any of the
/// differences.
line_start: BytePos,
/// Always 1, 2, or 4. Always as small as possible, while being big /// Always 1, 2, or 4. Always as small as possible, while being big
/// enough to hold the length of the longest line in the source file. /// enough to hold the length of the longest line in the source file.
/// The 1 case is by far the most common. /// The 1 case is by far the most common.
@ -1351,8 +1346,8 @@ pub struct SourceFile {
pub external_src: Lock<ExternalSource>, pub external_src: Lock<ExternalSource>,
/// The start position of this source in the `SourceMap`. /// The start position of this source in the `SourceMap`.
pub start_pos: BytePos, pub start_pos: BytePos,
/// The end position of this source in the `SourceMap`. /// The byte length of this source.
pub end_pos: BytePos, pub source_len: RelativeBytePos,
/// Locations of lines beginnings in the source code. /// Locations of lines beginnings in the source code.
pub lines: Lock<SourceFileLines>, pub lines: Lock<SourceFileLines>,
/// Locations of multi-byte characters in the source code. /// Locations of multi-byte characters in the source code.
@ -1375,7 +1370,7 @@ impl Clone for SourceFile {
src_hash: self.src_hash, src_hash: self.src_hash,
external_src: Lock::new(self.external_src.borrow().clone()), external_src: Lock::new(self.external_src.borrow().clone()),
start_pos: self.start_pos, start_pos: self.start_pos,
end_pos: self.end_pos, source_len: self.source_len,
lines: Lock::new(self.lines.borrow().clone()), lines: Lock::new(self.lines.borrow().clone()),
multibyte_chars: self.multibyte_chars.clone(), multibyte_chars: self.multibyte_chars.clone(),
non_narrow_chars: self.non_narrow_chars.clone(), non_narrow_chars: self.non_narrow_chars.clone(),
@ -1390,8 +1385,8 @@ impl<S: Encoder> Encodable<S> for SourceFile {
fn encode(&self, s: &mut S) { fn encode(&self, s: &mut S) {
self.name.encode(s); self.name.encode(s);
self.src_hash.encode(s); self.src_hash.encode(s);
self.start_pos.encode(s); // Do not encode `start_pos` as it's global state for this session.
self.end_pos.encode(s); self.source_len.encode(s);
// We are always in `Lines` form by the time we reach here. // We are always in `Lines` form by the time we reach here.
assert!(self.lines.borrow().is_lines()); assert!(self.lines.borrow().is_lines());
@ -1422,7 +1417,7 @@ impl<S: Encoder> Encodable<S> for SourceFile {
s.emit_u8(bytes_per_diff as u8); s.emit_u8(bytes_per_diff as u8);
// Encode the first element. // Encode the first element.
lines[0].encode(s); assert_eq!(lines[0], RelativeBytePos(0));
// Encode the difference list. // Encode the difference list.
let diff_iter = lines.array_windows().map(|&[fst, snd]| snd - fst); let diff_iter = lines.array_windows().map(|&[fst, snd]| snd - fst);
@ -1465,26 +1460,17 @@ impl<D: Decoder> Decodable<D> for SourceFile {
fn decode(d: &mut D) -> SourceFile { fn decode(d: &mut D) -> SourceFile {
let name: FileName = Decodable::decode(d); let name: FileName = Decodable::decode(d);
let src_hash: SourceFileHash = Decodable::decode(d); let src_hash: SourceFileHash = Decodable::decode(d);
let start_pos: BytePos = Decodable::decode(d); let source_len: RelativeBytePos = Decodable::decode(d);
let end_pos: BytePos = Decodable::decode(d);
let lines = { let lines = {
let num_lines: u32 = Decodable::decode(d); let num_lines: u32 = Decodable::decode(d);
if num_lines > 0 { if num_lines > 0 {
// Read the number of bytes used per diff. // Read the number of bytes used per diff.
let bytes_per_diff = d.read_u8() as usize; let bytes_per_diff = d.read_u8() as usize;
// Read the first element.
let line_start: BytePos = Decodable::decode(d);
// Read the difference list. // Read the difference list.
let num_diffs = num_lines as usize - 1; let num_diffs = num_lines as usize - 1;
let raw_diffs = d.read_raw_bytes(bytes_per_diff * num_diffs).to_vec(); let raw_diffs = d.read_raw_bytes(bytes_per_diff * num_diffs).to_vec();
SourceFileLines::Diffs(SourceFileDiffs { SourceFileLines::Diffs(SourceFileDiffs { bytes_per_diff, num_diffs, raw_diffs })
line_start,
bytes_per_diff,
num_diffs,
raw_diffs,
})
} else { } else {
SourceFileLines::Lines(vec![]) SourceFileLines::Lines(vec![])
} }
@ -1496,8 +1482,8 @@ impl<D: Decoder> Decodable<D> for SourceFile {
let cnum: CrateNum = Decodable::decode(d); let cnum: CrateNum = Decodable::decode(d);
SourceFile { SourceFile {
name, name,
start_pos, start_pos: BytePos::from_u32(0),
end_pos, source_len,
src: None, src: None,
src_hash, src_hash,
// Unused - the metadata decoder will construct // Unused - the metadata decoder will construct
@ -1523,63 +1509,58 @@ impl SourceFile {
pub fn new( pub fn new(
name: FileName, name: FileName,
mut src: String, mut src: String,
start_pos: BytePos,
hash_kind: SourceFileHashAlgorithm, hash_kind: SourceFileHashAlgorithm,
) -> Self { ) -> Result<Self, OffsetOverflowError> {
// Compute the file hash before any normalization. // Compute the file hash before any normalization.
let src_hash = SourceFileHash::new(hash_kind, &src); let src_hash = SourceFileHash::new(hash_kind, &src);
let normalized_pos = normalize_src(&mut src, start_pos); let normalized_pos = normalize_src(&mut src);
let name_hash = { let name_hash = {
let mut hasher: StableHasher = StableHasher::new(); let mut hasher: StableHasher = StableHasher::new();
name.hash(&mut hasher); name.hash(&mut hasher);
hasher.finish() hasher.finish()
}; };
let end_pos = start_pos.to_usize() + src.len(); let source_len = src.len();
assert!(end_pos <= u32::MAX as usize); let source_len = u32::try_from(source_len).map_err(|_| OffsetOverflowError)?;
let (lines, multibyte_chars, non_narrow_chars) = let (lines, multibyte_chars, non_narrow_chars) =
analyze_source_file::analyze_source_file(&src, start_pos); analyze_source_file::analyze_source_file(&src);
SourceFile { Ok(SourceFile {
name, name,
src: Some(Lrc::new(src)), src: Some(Lrc::new(src)),
src_hash, src_hash,
external_src: Lock::new(ExternalSource::Unneeded), external_src: Lock::new(ExternalSource::Unneeded),
start_pos, start_pos: BytePos::from_u32(0),
end_pos: Pos::from_usize(end_pos), source_len: RelativeBytePos::from_u32(source_len),
lines: Lock::new(SourceFileLines::Lines(lines)), lines: Lock::new(SourceFileLines::Lines(lines)),
multibyte_chars, multibyte_chars,
non_narrow_chars, non_narrow_chars,
normalized_pos, normalized_pos,
name_hash, name_hash,
cnum: LOCAL_CRATE, cnum: LOCAL_CRATE,
} })
} }
pub fn lines<F, R>(&self, f: F) -> R pub fn lines<F, R>(&self, f: F) -> R
where where
F: FnOnce(&[BytePos]) -> R, F: FnOnce(&[RelativeBytePos]) -> R,
{ {
let mut guard = self.lines.borrow_mut(); let mut guard = self.lines.borrow_mut();
match &*guard { match &*guard {
SourceFileLines::Lines(lines) => f(lines), SourceFileLines::Lines(lines) => f(lines),
SourceFileLines::Diffs(SourceFileDiffs { SourceFileLines::Diffs(SourceFileDiffs { bytes_per_diff, num_diffs, raw_diffs }) => {
mut line_start,
bytes_per_diff,
num_diffs,
raw_diffs,
}) => {
// Convert from "diffs" form to "lines" form. // Convert from "diffs" form to "lines" form.
let num_lines = num_diffs + 1; let num_lines = num_diffs + 1;
let mut lines = Vec::with_capacity(num_lines); let mut lines = Vec::with_capacity(num_lines);
let mut line_start = RelativeBytePos(0);
lines.push(line_start); lines.push(line_start);
assert_eq!(*num_diffs, raw_diffs.len() / bytes_per_diff); assert_eq!(*num_diffs, raw_diffs.len() / bytes_per_diff);
match bytes_per_diff { match bytes_per_diff {
1 => { 1 => {
lines.extend(raw_diffs.into_iter().map(|&diff| { lines.extend(raw_diffs.into_iter().map(|&diff| {
line_start = line_start + BytePos(diff as u32); line_start = line_start + RelativeBytePos(diff as u32);
line_start line_start
})); }));
} }
@ -1588,7 +1569,7 @@ impl SourceFile {
let pos = bytes_per_diff * i; let pos = bytes_per_diff * i;
let bytes = [raw_diffs[pos], raw_diffs[pos + 1]]; let bytes = [raw_diffs[pos], raw_diffs[pos + 1]];
let diff = u16::from_le_bytes(bytes); let diff = u16::from_le_bytes(bytes);
line_start = line_start + BytePos(diff as u32); line_start = line_start + RelativeBytePos(diff as u32);
line_start line_start
})); }));
} }
@ -1602,7 +1583,7 @@ impl SourceFile {
raw_diffs[pos + 3], raw_diffs[pos + 3],
]; ];
let diff = u32::from_le_bytes(bytes); let diff = u32::from_le_bytes(bytes);
line_start = line_start + BytePos(diff); line_start = line_start + RelativeBytePos(diff);
line_start line_start
})); }));
} }
@ -1617,8 +1598,10 @@ impl SourceFile {
/// Returns the `BytePos` of the beginning of the current line. /// Returns the `BytePos` of the beginning of the current line.
pub fn line_begin_pos(&self, pos: BytePos) -> BytePos { pub fn line_begin_pos(&self, pos: BytePos) -> BytePos {
let pos = self.relative_position(pos);
let line_index = self.lookup_line(pos).unwrap(); let line_index = self.lookup_line(pos).unwrap();
self.lines(|lines| lines[line_index]) let line_start_pos = self.lines(|lines| lines[line_index]);
self.absolute_position(line_start_pos)
} }
/// Add externally loaded source. /// Add externally loaded source.
@ -1643,7 +1626,7 @@ impl SourceFile {
if let Some(mut src) = src { if let Some(mut src) = src {
// The src_hash needs to be computed on the pre-normalized src. // The src_hash needs to be computed on the pre-normalized src.
if self.src_hash.matches(&src) { if self.src_hash.matches(&src) {
normalize_src(&mut src, BytePos::from_usize(0)); normalize_src(&mut src);
*src_kind = ExternalSourceKind::Present(Lrc::new(src)); *src_kind = ExternalSourceKind::Present(Lrc::new(src));
return true; return true;
} }
@ -1676,8 +1659,7 @@ impl SourceFile {
let begin = { let begin = {
let line = self.lines(|lines| lines.get(line_number).copied())?; let line = self.lines(|lines| lines.get(line_number).copied())?;
let begin: BytePos = line - self.start_pos; line.to_usize()
begin.to_usize()
}; };
if let Some(ref src) = self.src { if let Some(ref src) = self.src {
@ -1703,25 +1685,41 @@ impl SourceFile {
self.lines(|lines| lines.len()) self.lines(|lines| lines.len())
} }
#[inline]
pub fn absolute_position(&self, pos: RelativeBytePos) -> BytePos {
BytePos::from_u32(pos.to_u32() + self.start_pos.to_u32())
}
#[inline]
pub fn relative_position(&self, pos: BytePos) -> RelativeBytePos {
RelativeBytePos::from_u32(pos.to_u32() - self.start_pos.to_u32())
}
#[inline]
pub fn end_position(&self) -> BytePos {
self.absolute_position(self.source_len)
}
/// Finds the line containing the given position. The return value is the /// Finds the line containing the given position. The return value is the
/// index into the `lines` array of this `SourceFile`, not the 1-based line /// index into the `lines` array of this `SourceFile`, not the 1-based line
/// number. If the source_file is empty or the position is located before the /// number. If the source_file is empty or the position is located before the
/// first line, `None` is returned. /// first line, `None` is returned.
pub fn lookup_line(&self, pos: BytePos) -> Option<usize> { pub fn lookup_line(&self, pos: RelativeBytePos) -> Option<usize> {
self.lines(|lines| lines.partition_point(|x| x <= &pos).checked_sub(1)) self.lines(|lines| lines.partition_point(|x| x <= &pos).checked_sub(1))
} }
pub fn line_bounds(&self, line_index: usize) -> Range<BytePos> { pub fn line_bounds(&self, line_index: usize) -> Range<BytePos> {
if self.is_empty() { if self.is_empty() {
return self.start_pos..self.end_pos; return self.start_pos..self.start_pos;
} }
self.lines(|lines| { self.lines(|lines| {
assert!(line_index < lines.len()); assert!(line_index < lines.len());
if line_index == (lines.len() - 1) { if line_index == (lines.len() - 1) {
lines[line_index]..self.end_pos self.absolute_position(lines[line_index])..self.end_position()
} else { } else {
lines[line_index]..lines[line_index + 1] self.absolute_position(lines[line_index])
..self.absolute_position(lines[line_index + 1])
} }
}) })
} }
@ -1732,17 +1730,19 @@ impl SourceFile {
/// returns true still contain one byte position according to this function. /// returns true still contain one byte position according to this function.
#[inline] #[inline]
pub fn contains(&self, byte_pos: BytePos) -> bool { pub fn contains(&self, byte_pos: BytePos) -> bool {
byte_pos >= self.start_pos && byte_pos <= self.end_pos byte_pos >= self.start_pos && byte_pos <= self.end_position()
} }
#[inline] #[inline]
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.start_pos == self.end_pos self.source_len.to_u32() == 0
} }
/// Calculates the original byte position relative to the start of the file /// Calculates the original byte position relative to the start of the file
/// based on the given byte position. /// based on the given byte position.
pub fn original_relative_byte_pos(&self, pos: BytePos) -> BytePos { pub fn original_relative_byte_pos(&self, pos: BytePos) -> RelativeBytePos {
let pos = self.relative_position(pos);
// Diff before any records is 0. Otherwise use the previously recorded // Diff before any records is 0. Otherwise use the previously recorded
// diff as that applies to the following characters until a new diff // diff as that applies to the following characters until a new diff
// is recorded. // is recorded.
@ -1752,7 +1752,7 @@ impl SourceFile {
Err(i) => self.normalized_pos[i - 1].diff, Err(i) => self.normalized_pos[i - 1].diff,
}; };
BytePos::from_u32(pos.0 - self.start_pos.0 + diff) RelativeBytePos::from_u32(pos.0 + diff)
} }
/// Calculates a normalized byte position from a byte offset relative to the /// Calculates a normalized byte position from a byte offset relative to the
@ -1777,8 +1777,8 @@ impl SourceFile {
BytePos::from_u32(self.start_pos.0 + offset - diff) BytePos::from_u32(self.start_pos.0 + offset - diff)
} }
/// Converts an absolute `BytePos` to a `CharPos` relative to the `SourceFile`. /// Converts an relative `RelativeBytePos` to a `CharPos` relative to the `SourceFile`.
pub fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos { fn bytepos_to_file_charpos(&self, bpos: RelativeBytePos) -> CharPos {
// The number of extra bytes due to multibyte chars in the `SourceFile`. // The number of extra bytes due to multibyte chars in the `SourceFile`.
let mut total_extra_bytes = 0; let mut total_extra_bytes = 0;
@ -1796,13 +1796,13 @@ impl SourceFile {
} }
} }
assert!(self.start_pos.to_u32() + total_extra_bytes <= bpos.to_u32()); assert!(total_extra_bytes <= bpos.to_u32());
CharPos(bpos.to_usize() - self.start_pos.to_usize() - total_extra_bytes as usize) CharPos(bpos.to_usize() - total_extra_bytes as usize)
} }
/// Looks up the file's (1-based) line number and (0-based `CharPos`) column offset, for a /// Looks up the file's (1-based) line number and (0-based `CharPos`) column offset, for a
/// given `BytePos`. /// given `RelativeBytePos`.
pub fn lookup_file_pos(&self, pos: BytePos) -> (usize, CharPos) { fn lookup_file_pos(&self, pos: RelativeBytePos) -> (usize, CharPos) {
let chpos = self.bytepos_to_file_charpos(pos); let chpos = self.bytepos_to_file_charpos(pos);
match self.lookup_line(pos) { match self.lookup_line(pos) {
Some(a) => { Some(a) => {
@ -1823,6 +1823,7 @@ impl SourceFile {
/// Looks up the file's (1-based) line number, (0-based `CharPos`) column offset, and (0-based) /// Looks up the file's (1-based) line number, (0-based `CharPos`) column offset, and (0-based)
/// column offset when displayed, for a given `BytePos`. /// column offset when displayed, for a given `BytePos`.
pub fn lookup_file_pos_with_col_display(&self, pos: BytePos) -> (usize, CharPos, usize) { pub fn lookup_file_pos_with_col_display(&self, pos: BytePos) -> (usize, CharPos, usize) {
let pos = self.relative_position(pos);
let (line, col_or_chpos) = self.lookup_file_pos(pos); let (line, col_or_chpos) = self.lookup_file_pos(pos);
if line > 0 { if line > 0 {
let col = col_or_chpos; let col = col_or_chpos;
@ -1861,16 +1862,10 @@ impl SourceFile {
} }
/// Normalizes the source code and records the normalizations. /// Normalizes the source code and records the normalizations.
fn normalize_src(src: &mut String, start_pos: BytePos) -> Vec<NormalizedPos> { fn normalize_src(src: &mut String) -> Vec<NormalizedPos> {
let mut normalized_pos = vec![]; let mut normalized_pos = vec![];
remove_bom(src, &mut normalized_pos); remove_bom(src, &mut normalized_pos);
normalize_newlines(src, &mut normalized_pos); normalize_newlines(src, &mut normalized_pos);
// Offset all the positions by start_pos to match the final file positions.
for np in &mut normalized_pos {
np.pos.0 += start_pos.0;
}
normalized_pos normalized_pos
} }
@ -1878,7 +1873,7 @@ fn normalize_src(src: &mut String, start_pos: BytePos) -> Vec<NormalizedPos> {
fn remove_bom(src: &mut String, normalized_pos: &mut Vec<NormalizedPos>) { fn remove_bom(src: &mut String, normalized_pos: &mut Vec<NormalizedPos>) {
if src.starts_with('\u{feff}') { if src.starts_with('\u{feff}') {
src.drain(..3); src.drain(..3);
normalized_pos.push(NormalizedPos { pos: BytePos(0), diff: 3 }); normalized_pos.push(NormalizedPos { pos: RelativeBytePos(0), diff: 3 });
} }
} }
@ -1913,7 +1908,7 @@ fn normalize_newlines(src: &mut String, normalized_pos: &mut Vec<NormalizedPos>)
cursor += idx - gap_len; cursor += idx - gap_len;
gap_len += 1; gap_len += 1;
normalized_pos.push(NormalizedPos { normalized_pos.push(NormalizedPos {
pos: BytePos::from_usize(cursor + 1), pos: RelativeBytePos::from_usize(cursor + 1),
diff: original_gap + gap_len as u32, diff: original_gap + gap_len as u32,
}); });
} }
@ -2015,6 +2010,10 @@ impl_pos! {
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
pub struct BytePos(pub u32); pub struct BytePos(pub u32);
/// A byte offset relative to file beginning.
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
pub struct RelativeBytePos(pub u32);
/// A character offset. /// A character offset.
/// ///
/// Because of multibyte UTF-8 characters, a byte offset /// Because of multibyte UTF-8 characters, a byte offset
@ -2036,6 +2035,24 @@ impl<D: Decoder> Decodable<D> for BytePos {
} }
} }
impl<H: HashStableContext> HashStable<H> for RelativeBytePos {
fn hash_stable(&self, hcx: &mut H, hasher: &mut StableHasher) {
self.0.hash_stable(hcx, hasher);
}
}
impl<S: Encoder> Encodable<S> for RelativeBytePos {
fn encode(&self, s: &mut S) {
s.emit_u32(self.0);
}
}
impl<D: Decoder> Decodable<D> for RelativeBytePos {
fn decode(d: &mut D) -> RelativeBytePos {
RelativeBytePos(d.read_u32())
}
}
// _____________________________________________________________________________ // _____________________________________________________________________________
// Loc, SourceFileAndLine, SourceFileAndBytePos // Loc, SourceFileAndLine, SourceFileAndBytePos
// //

149
compiler/rustc_span/src/source_map.rs
View File

@ -14,13 +14,10 @@ pub use crate::*;
use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::stable_hasher::{Hash128, Hash64, StableHasher}; use rustc_data_structures::stable_hasher::{Hash128, Hash64, StableHasher};
use rustc_data_structures::sync::{ use rustc_data_structures::sync::{IntoDynSyncSend, Lrc, MappedReadGuard, ReadGuard, RwLock};
AtomicU32, IntoDynSyncSend, Lrc, MappedReadGuard, ReadGuard, RwLock,
};
use std::cmp; use std::cmp;
use std::hash::Hash; use std::hash::Hash;
use std::path::{self, Path, PathBuf}; use std::path::{self, Path, PathBuf};
use std::sync::atomic::Ordering;
use std::fs; use std::fs;
use std::io; use std::io;
@ -187,9 +184,6 @@ pub(super) struct SourceMapFiles {
} }
pub struct SourceMap { pub struct SourceMap {
/// The address space below this value is currently used by the files in the source map.
used_address_space: AtomicU32,
files: RwLock<SourceMapFiles>, files: RwLock<SourceMapFiles>,
file_loader: IntoDynSyncSend<Box<dyn FileLoader + Sync + Send>>, file_loader: IntoDynSyncSend<Box<dyn FileLoader + Sync + Send>>,
// This is used to apply the file path remapping as specified via // This is used to apply the file path remapping as specified via
@ -215,7 +209,6 @@ impl SourceMap {
hash_kind: SourceFileHashAlgorithm, hash_kind: SourceFileHashAlgorithm,
) -> SourceMap { ) -> SourceMap {
SourceMap { SourceMap {
used_address_space: AtomicU32::new(0),
files: Default::default(), files: Default::default(),
file_loader: IntoDynSyncSend(file_loader), file_loader: IntoDynSyncSend(file_loader),
path_mapping, path_mapping,
@ -267,26 +260,26 @@ impl SourceMap {
self.files.borrow().stable_id_to_source_file.get(&stable_id).cloned() self.files.borrow().stable_id_to_source_file.get(&stable_id).cloned()
} }
fn allocate_address_space(&self, size: usize) -> Result<usize, OffsetOverflowError> { fn register_source_file(
let size = u32::try_from(size).map_err(|_| OffsetOverflowError)?; &self,
file_id: StableSourceFileId,
mut file: SourceFile,
) -> Result<Lrc<SourceFile>, OffsetOverflowError> {
let mut files = self.files.borrow_mut();
loop { file.start_pos = BytePos(if let Some(last_file) = files.source_files.last() {
let current = self.used_address_space.load(Ordering::Relaxed); // Add one so there is some space between files. This lets us distinguish
let next = current // positions in the `SourceMap`, even in the presence of zero-length files.
.checked_add(size) last_file.end_position().0.checked_add(1).ok_or(OffsetOverflowError)?
// Add one so there is some space between files. This lets us distinguish } else {
// positions in the `SourceMap`, even in the presence of zero-length files. 0
.and_then(|next| next.checked_add(1)) });
.ok_or(OffsetOverflowError)?;
if self let file = Lrc::new(file);
.used_address_space files.source_files.push(file.clone());
.compare_exchange(current, next, Ordering::Relaxed, Ordering::Relaxed) files.stable_id_to_source_file.insert(file_id, file.clone());
.is_ok()
{ Ok(file)
return Ok(usize::try_from(current).unwrap());
}
}
} }
/// Creates a new `SourceFile`. /// Creates a new `SourceFile`.
@ -310,32 +303,18 @@ impl SourceMap {
let (filename, _) = self.path_mapping.map_filename_prefix(&filename); let (filename, _) = self.path_mapping.map_filename_prefix(&filename);
let file_id = StableSourceFileId::new_from_name(&filename, LOCAL_CRATE); let file_id = StableSourceFileId::new_from_name(&filename, LOCAL_CRATE);
match self.source_file_by_stable_id(file_id) {
let lrc_sf = match self.source_file_by_stable_id(file_id) { Some(lrc_sf) => Ok(lrc_sf),
Some(lrc_sf) => lrc_sf,
None => { None => {
let start_pos = self.allocate_address_space(src.len())?; let source_file = SourceFile::new(filename, src, self.hash_kind)?;
let source_file = Lrc::new(SourceFile::new(
filename,
src,
Pos::from_usize(start_pos),
self.hash_kind,
));
// Let's make sure the file_id we generated above actually matches // Let's make sure the file_id we generated above actually matches
// the ID we generate for the SourceFile we just created. // the ID we generate for the SourceFile we just created.
debug_assert_eq!(StableSourceFileId::new(&source_file), file_id); debug_assert_eq!(StableSourceFileId::new(&source_file), file_id);
let mut files = self.files.borrow_mut(); self.register_source_file(file_id, source_file)
files.source_files.push(source_file.clone());
files.stable_id_to_source_file.insert(file_id, source_file.clone());
source_file
} }
}; }
Ok(lrc_sf)
} }
/// Allocates a new `SourceFile` representing a source file from an external /// Allocates a new `SourceFile` representing a source file from an external
@ -347,53 +326,17 @@ impl SourceMap {
filename: FileName, filename: FileName,
src_hash: SourceFileHash, src_hash: SourceFileHash,
name_hash: Hash128, name_hash: Hash128,
source_len: usize, source_len: u32,
cnum: CrateNum, cnum: CrateNum,
file_local_lines: Lock<SourceFileLines>, file_local_lines: Lock<SourceFileLines>,
mut file_local_multibyte_chars: Vec<MultiByteChar>, multibyte_chars: Vec<MultiByteChar>,
mut file_local_non_narrow_chars: Vec<NonNarrowChar>, non_narrow_chars: Vec<NonNarrowChar>,
mut file_local_normalized_pos: Vec<NormalizedPos>, normalized_pos: Vec<NormalizedPos>,
original_start_pos: BytePos,
metadata_index: u32, metadata_index: u32,
) -> Lrc<SourceFile> { ) -> Lrc<SourceFile> {
let start_pos = self let source_len = RelativeBytePos::from_u32(source_len);
.allocate_address_space(source_len)
.expect("not enough address space for imported source file");
let end_pos = Pos::from_usize(start_pos + source_len); let source_file = SourceFile {
let start_pos = Pos::from_usize(start_pos);
// Translate these positions into the new global frame of reference,
// now that the offset of the SourceFile is known.
//
// These are all unsigned values. `original_start_pos` may be larger or
// smaller than `start_pos`, but `pos` is always larger than both.
// Therefore, `(pos - original_start_pos) + start_pos` won't overflow
// but `start_pos - original_start_pos` might. So we use the former
// form rather than pre-computing the offset into a local variable. The
// compiler backend can optimize away the repeated computations in a
// way that won't trigger overflow checks.
match &mut *file_local_lines.borrow_mut() {
SourceFileLines::Lines(lines) => {
for pos in lines {
*pos = (*pos - original_start_pos) + start_pos;
}
}
SourceFileLines::Diffs(SourceFileDiffs { line_start, .. }) => {
*line_start = (*line_start - original_start_pos) + start_pos;
}
}
for mbc in &mut file_local_multibyte_chars {
mbc.pos = (mbc.pos - original_start_pos) + start_pos;
}
for swc in &mut file_local_non_narrow_chars {
*swc = (*swc - original_start_pos) + start_pos;
}
for nc in &mut file_local_normalized_pos {
nc.pos = (nc.pos - original_start_pos) + start_pos;
}
let source_file = Lrc::new(SourceFile {
name: filename, name: filename,
src: None, src: None,
src_hash, src_hash,
@ -401,24 +344,19 @@ impl SourceMap {
kind: ExternalSourceKind::AbsentOk, kind: ExternalSourceKind::AbsentOk,
metadata_index, metadata_index,
}), }),
start_pos, start_pos: BytePos(0),
end_pos, source_len,
lines: file_local_lines, lines: file_local_lines,
multibyte_chars: file_local_multibyte_chars, multibyte_chars,
non_narrow_chars: file_local_non_narrow_chars, non_narrow_chars,
normalized_pos: file_local_normalized_pos, normalized_pos,
name_hash, name_hash,
cnum, cnum,
}); };
let mut files = self.files.borrow_mut(); let file_id = StableSourceFileId::new(&source_file);
self.register_source_file(file_id, source_file)
files.source_files.push(source_file.clone()); .expect("not enough address space for imported source file")
files
.stable_id_to_source_file
.insert(StableSourceFileId::new(&source_file), source_file.clone());
source_file
} }
/// If there is a doctest offset, applies it to the line. /// If there is a doctest offset, applies it to the line.
@ -452,6 +390,7 @@ impl SourceMap {
pub fn lookup_line(&self, pos: BytePos) -> Result<SourceFileAndLine, Lrc<SourceFile>> { pub fn lookup_line(&self, pos: BytePos) -> Result<SourceFileAndLine, Lrc<SourceFile>> {
let f = self.lookup_source_file(pos); let f = self.lookup_source_file(pos);
let pos = f.relative_position(pos);
match f.lookup_line(pos) { match f.lookup_line(pos) {
Some(line) => Ok(SourceFileAndLine { sf: f, line }), Some(line) => Ok(SourceFileAndLine { sf: f, line }),
None => Err(f), None => Err(f),
@ -547,7 +486,9 @@ impl SourceMap {
return true; return true;
} }
let f = (*self.files.borrow().source_files)[lo].clone(); let f = (*self.files.borrow().source_files)[lo].clone();
f.lookup_line(sp.lo()) != f.lookup_line(sp.hi()) let lo = f.relative_position(sp.lo());
let hi = f.relative_position(sp.hi());
f.lookup_line(lo) != f.lookup_line(hi)
} }
#[instrument(skip(self), level = "trace")] #[instrument(skip(self), level = "trace")]
@ -627,7 +568,7 @@ impl SourceMap {
let start_index = local_begin.pos.to_usize(); let start_index = local_begin.pos.to_usize();
let end_index = local_end.pos.to_usize(); let end_index = local_end.pos.to_usize();
let source_len = (local_begin.sf.end_pos - local_begin.sf.start_pos).to_usize(); let source_len = local_begin.sf.source_len.to_usize();
if start_index > end_index || end_index > source_len { if start_index > end_index || end_index > source_len {
return Err(SpanSnippetError::MalformedForSourcemap(MalformedSourceMapPositions { return Err(SpanSnippetError::MalformedForSourcemap(MalformedSourceMapPositions {
@ -1034,7 +975,7 @@ impl SourceMap {
return 1; return 1;
} }
let source_len = (local_begin.sf.end_pos - local_begin.sf.start_pos).to_usize(); let source_len = local_begin.sf.source_len.to_usize();
debug!("source_len=`{:?}`", source_len); debug!("source_len=`{:?}`", source_len);
// Ensure indexes are also not malformed. // Ensure indexes are also not malformed.
if start_index > end_index || end_index > source_len - 1 { if start_index > end_index || end_index > source_len - 1 {

7
compiler/rustc_span/src/source_map/tests.rs
View File

@ -50,6 +50,7 @@ impl SourceMap {
fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos { fn bytepos_to_file_charpos(&self, bpos: BytePos) -> CharPos {
let idx = self.lookup_source_file_idx(bpos); let idx = self.lookup_source_file_idx(bpos);
let sf = &(*self.files.borrow().source_files)[idx]; let sf = &(*self.files.borrow().source_files)[idx];
let bpos = sf.relative_position(bpos);
sf.bytepos_to_file_charpos(bpos) sf.bytepos_to_file_charpos(bpos)
} }
} }
@ -230,8 +231,7 @@ fn t10() {
let SourceFile { let SourceFile {
name, name,
src_hash, src_hash,
start_pos, source_len,
end_pos,
lines, lines,
multibyte_chars, multibyte_chars,
non_narrow_chars, non_narrow_chars,
@ -244,13 +244,12 @@ fn t10() {
name, name,
src_hash, src_hash,
name_hash, name_hash,
(end_pos - start_pos).to_usize(), source_len.to_u32(),
CrateNum::new(0), CrateNum::new(0),
lines, lines,
multibyte_chars, multibyte_chars,
non_narrow_chars, non_narrow_chars,
normalized_pos, normalized_pos,
start_pos,
0, 0,
); );

29
compiler/rustc_span/src/tests.rs
View File

@ -3,24 +3,23 @@ use super::*;
#[test] #[test]
fn test_lookup_line() { fn test_lookup_line() {
let source = "abcdefghijklm\nabcdefghij\n...".to_owned(); let source = "abcdefghijklm\nabcdefghij\n...".to_owned();
let sf = SourceFile::new( let mut sf =
FileName::Anon(Hash64::ZERO), SourceFile::new(FileName::Anon(Hash64::ZERO), source, SourceFileHashAlgorithm::Sha256)
source, .unwrap();
BytePos(3), sf.start_pos = BytePos(3);
SourceFileHashAlgorithm::Sha256, sf.lines(|lines| {
); assert_eq!(lines, &[RelativeBytePos(0), RelativeBytePos(14), RelativeBytePos(25)])
sf.lines(|lines| assert_eq!(lines, &[BytePos(3), BytePos(17), BytePos(28)])); });
assert_eq!(sf.lookup_line(BytePos(0)), None); assert_eq!(sf.lookup_line(RelativeBytePos(0)), Some(0));
assert_eq!(sf.lookup_line(BytePos(3)), Some(0)); assert_eq!(sf.lookup_line(RelativeBytePos(1)), Some(0));
assert_eq!(sf.lookup_line(BytePos(4)), Some(0));
assert_eq!(sf.lookup_line(BytePos(16)), Some(0)); assert_eq!(sf.lookup_line(RelativeBytePos(13)), Some(0));
assert_eq!(sf.lookup_line(BytePos(17)), Some(1)); assert_eq!(sf.lookup_line(RelativeBytePos(14)), Some(1));
assert_eq!(sf.lookup_line(BytePos(18)), Some(1)); assert_eq!(sf.lookup_line(RelativeBytePos(15)), Some(1));
assert_eq!(sf.lookup_line(BytePos(28)), Some(2)); assert_eq!(sf.lookup_line(RelativeBytePos(25)), Some(2));
assert_eq!(sf.lookup_line(BytePos(29)), Some(2)); assert_eq!(sf.lookup_line(RelativeBytePos(26)), Some(2));
} }
#[test] #[test]

2
src/librustdoc/html/sources.rs
View File

@ -134,7 +134,7 @@ impl DocVisitor for SourceCollector<'_, '_> {
let filename = span.filename(sess); let filename = span.filename(sess);
let span = span.inner(); let span = span.inner();
let pos = sess.source_map().lookup_source_file(span.lo()); let pos = sess.source_map().lookup_source_file(span.lo());
let file_span = span.with_lo(pos.start_pos).with_hi(pos.end_pos); let file_span = span.with_lo(pos.start_pos).with_hi(pos.end_position());
// If it turns out that we couldn't read this file, then we probably // If it turns out that we couldn't read this file, then we probably
// can't read any of the files (generating html output from json or // can't read any of the files (generating html output from json or
// something like that), so just don't include sources for the // something like that), so just don't include sources for the

2
src/librustdoc/scrape_examples.rs
View File

@ -73,7 +73,7 @@ pub(crate) struct SyntaxRange {
impl SyntaxRange { impl SyntaxRange {
fn new(span: rustc_span::Span, file: &SourceFile) -> Option<Self> { fn new(span: rustc_span::Span, file: &SourceFile) -> Option<Self> {
let get_pos = |bytepos: BytePos| file.original_relative_byte_pos(bytepos).0; let get_pos = |bytepos: BytePos| file.original_relative_byte_pos(bytepos).0;
let get_line = |bytepos: BytePos| file.lookup_line(bytepos); let get_line = |bytepos: BytePos| file.lookup_line(file.relative_position(bytepos));
Some(SyntaxRange { Some(SyntaxRange {
byte_span: (get_pos(span.lo()), get_pos(span.hi())), byte_span: (get_pos(span.lo()), get_pos(span.hi())),

26
src/tools/clippy/clippy_lints/src/undocumented_unsafe_blocks.rs
View File

@ -12,7 +12,7 @@ use rustc_lexer::{tokenize, TokenKind};
use rustc_lint::{LateContext, LateLintPass, LintContext}; use rustc_lint::{LateContext, LateLintPass, LintContext};
use rustc_middle::lint::in_external_macro; use rustc_middle::lint::in_external_macro;
use rustc_session::{declare_tool_lint, impl_lint_pass}; use rustc_session::{declare_tool_lint, impl_lint_pass};
use rustc_span::{BytePos, Pos, Span, SyntaxContext}; use rustc_span::{BytePos, Pos, RelativeBytePos, Span, SyntaxContext};
declare_clippy_lint! { declare_clippy_lint! {
/// ### What it does /// ### What it does
@ -514,7 +514,7 @@ fn item_has_safety_comment(cx: &LateContext<'_>, item: &hir::Item<'_>) -> HasSaf
match text_has_safety_comment( match text_has_safety_comment(
src, src,
&lines[comment_start_line.line + 1..=unsafe_line.line], &lines[comment_start_line.line + 1..=unsafe_line.line],
unsafe_line.sf.start_pos.to_usize(), unsafe_line.sf.start_pos,
) { ) {
Some(b) => HasSafetyComment::Yes(b), Some(b) => HasSafetyComment::Yes(b),
None => HasSafetyComment::No, None => HasSafetyComment::No,
@ -558,7 +558,7 @@ fn stmt_has_safety_comment(cx: &LateContext<'_>, span: Span, hir_id: HirId) -> H
match text_has_safety_comment( match text_has_safety_comment(
src, src,
&lines[comment_start_line.line + 1..=unsafe_line.line], &lines[comment_start_line.line + 1..=unsafe_line.line],
unsafe_line.sf.start_pos.to_usize(), unsafe_line.sf.start_pos,
) { ) {
Some(b) => HasSafetyComment::Yes(b), Some(b) => HasSafetyComment::Yes(b),
None => HasSafetyComment::No, None => HasSafetyComment::No,
@ -619,7 +619,7 @@ fn span_from_macro_expansion_has_safety_comment(cx: &LateContext<'_>, span: Span
match text_has_safety_comment( match text_has_safety_comment(
src, src,
&lines[macro_line.line + 1..=unsafe_line.line], &lines[macro_line.line + 1..=unsafe_line.line],
unsafe_line.sf.start_pos.to_usize(), unsafe_line.sf.start_pos,
) { ) {
Some(b) => HasSafetyComment::Yes(b), Some(b) => HasSafetyComment::Yes(b),
None => HasSafetyComment::No, None => HasSafetyComment::No,
@ -675,7 +675,7 @@ fn span_in_body_has_safety_comment(cx: &LateContext<'_>, span: Span) -> bool {
body_line.line < unsafe_line.line && text_has_safety_comment( body_line.line < unsafe_line.line && text_has_safety_comment(
src, src,
&lines[body_line.line + 1..=unsafe_line.line], &lines[body_line.line + 1..=unsafe_line.line],
unsafe_line.sf.start_pos.to_usize(), unsafe_line.sf.start_pos,
).is_some() ).is_some()
}) })
} else { } else {
@ -688,13 +688,13 @@ fn span_in_body_has_safety_comment(cx: &LateContext<'_>, span: Span) -> bool {
} }
/// Checks if the given text has a safety comment for the immediately proceeding line. /// Checks if the given text has a safety comment for the immediately proceeding line.
fn text_has_safety_comment(src: &str, line_starts: &[BytePos], offset: usize) -> Option<BytePos> { fn text_has_safety_comment(src: &str, line_starts: &[RelativeBytePos], start_pos: BytePos) -> Option<BytePos> {
let mut lines = line_starts let mut lines = line_starts
.array_windows::<2>() .array_windows::<2>()
.rev() .rev()
.map_while(|[start, end]| { .map_while(|[start, end]| {
let start = start.to_usize() - offset; let start = start.to_usize();
let end = end.to_usize() - offset; let end = end.to_usize();
let text = src.get(start..end)?; let text = src.get(start..end)?;
let trimmed = text.trim_start(); let trimmed = text.trim_start();
Some((start + (text.len() - trimmed.len()), trimmed)) Some((start + (text.len() - trimmed.len()), trimmed))
@ -709,9 +709,7 @@ fn text_has_safety_comment(src: &str, line_starts: &[BytePos], offset: usize) ->
let (mut line, mut line_start) = (line, line_start); let (mut line, mut line_start) = (line, line_start);
loop { loop {
if line.to_ascii_uppercase().contains("SAFETY:") { if line.to_ascii_uppercase().contains("SAFETY:") {
return Some(BytePos( return Some(start_pos + BytePos(u32::try_from(line_start).unwrap()));
u32::try_from(line_start).unwrap() + u32::try_from(offset).unwrap(),
));
} }
match lines.next() { match lines.next() {
Some((s, x)) if x.starts_with("//") => (line, line_start) = (x, s), Some((s, x)) if x.starts_with("//") => (line, line_start) = (x, s),
@ -724,15 +722,13 @@ fn text_has_safety_comment(src: &str, line_starts: &[BytePos], offset: usize) ->
let (mut line_start, mut line) = (line_start, line); let (mut line_start, mut line) = (line_start, line);
loop { loop {
if line.starts_with("/*") { if line.starts_with("/*") {
let src = &src[line_start..line_starts.last().unwrap().to_usize() - offset]; let src = &src[line_start..line_starts.last().unwrap().to_usize()];
let mut tokens = tokenize(src); let mut tokens = tokenize(src);
return (src[..tokens.next().unwrap().len as usize] return (src[..tokens.next().unwrap().len as usize]
.to_ascii_uppercase() .to_ascii_uppercase()
.contains("SAFETY:") .contains("SAFETY:")
&& tokens.all(|t| t.kind == TokenKind::Whitespace)) && tokens.all(|t| t.kind == TokenKind::Whitespace))
.then_some(BytePos( .then_some(start_pos + BytePos(u32::try_from(line_start).unwrap()));
u32::try_from(line_start).unwrap() + u32::try_from(offset).unwrap(),
));
} }
match lines.next() { match lines.next() {
Some(x) => (line_start, line) = x, Some(x) => (line_start, line) = x,

8
src/tools/clippy/clippy_utils/src/source.rs
View File

@ -8,7 +8,7 @@ use rustc_hir::{BlockCheckMode, Expr, ExprKind, UnsafeSource};
use rustc_lint::{LateContext, LintContext}; use rustc_lint::{LateContext, LintContext};
use rustc_session::Session; use rustc_session::Session;
use rustc_span::source_map::{original_sp, SourceMap}; use rustc_span::source_map::{original_sp, SourceMap};
use rustc_span::{hygiene, BytePos, Pos, SourceFile, Span, SpanData, SyntaxContext, DUMMY_SP}; use rustc_span::{hygiene, BytePos, SourceFileAndLine, Pos, SourceFile, Span, SpanData, SyntaxContext, DUMMY_SP};
use std::borrow::Cow; use std::borrow::Cow;
use std::ops::Range; use std::ops::Range;
@ -117,9 +117,9 @@ fn first_char_in_first_line<T: LintContext>(cx: &T, span: Span) -> Option<BytePo
/// ``` /// ```
fn line_span<T: LintContext>(cx: &T, span: Span) -> Span { fn line_span<T: LintContext>(cx: &T, span: Span) -> Span {
let span = original_sp(span, DUMMY_SP); let span = original_sp(span, DUMMY_SP);
let source_map_and_line = cx.sess().source_map().lookup_line(span.lo()).unwrap(); let SourceFileAndLine { sf, line } = cx.sess().source_map().lookup_line(span.lo()).unwrap();
let line_no = source_map_and_line.line; let line_start = sf.lines(|lines| lines[line]);
let line_start = source_map_and_line.sf.lines(|lines| lines[line_no]); let line_start = sf.absolute_position(line_start);
span.with_lo(line_start) span.with_lo(line_start)
} }

2
src/tools/rustfmt/src/parse/session.rs
View File

@ -268,7 +268,7 @@ impl ParseSess {
let source_file = self.parse_sess.source_map().lookup_char_pos(span.lo()).file; let source_file = self.parse_sess.source_map().lookup_char_pos(span.lo()).file;
SnippetProvider::new( SnippetProvider::new(
source_file.start_pos, source_file.start_pos,
source_file.end_pos, source_file.end_position(),
Lrc::clone(source_file.src.as_ref().unwrap()), Lrc::clone(source_file.src.as_ref().unwrap()),
) )
} }