nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-27 17:24:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lexer: Fix offset handling in get_str_from()

Browse Source 
As the comment said, the subtraction is bogus for multibyte characters.
Fortunately, we can just use last_pos instead of pos to get the correct
position without any subtraction hackery.
This commit is contained in:
Björn Steinbrink 2013-06-07 20:00:37 +02:00
parent b8cf2f8056
commit 43cae88079
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/libsyntax/parse/comments.rs
View File

@ -347,7 +347,7 @@ pub fn gather_comments_and_literals(span_diagnostic:
}
let bstart = rdr.pos;
let bstart = rdr.last_pos;
rdr.next_token();
//discard, and look ahead; we're working with internal state
let TokenAndSpan {tok: tok, sp: sp} = rdr.peek();

16
src/libsyntax/parse/lexer.rs
View File

@ -161,22 +161,20 @@ fn string_advance_token(r: @mut StringReader) {
}
}
fn byte_offset(rdr: &StringReader) -> BytePos {
(rdr.pos - rdr.filemap.start_pos)
fn byte_offset(rdr: &StringReader, pos: BytePos) -> BytePos {
(pos - rdr.filemap.start_pos)
}
pub fn get_str_from(rdr: @mut StringReader, start: BytePos) -> ~str {
// I'm pretty skeptical about this subtraction. What if there's a
// multi-byte character before the mark?
return str::slice(*rdr.src, start.to_uint() - 1u,
byte_offset(rdr).to_uint() - 1u).to_owned();
return str::slice(*rdr.src, start.to_uint(),
byte_offset(rdr, rdr.last_pos).to_uint()).to_owned();
}
// EFFECT: advance the StringReader by one character. If a newline is
// discovered, add it to the FileMap's list of line start offsets.
pub fn bump(rdr: &mut StringReader) {
rdr.last_pos = rdr.pos;
let current_byte_offset = byte_offset(rdr).to_uint();;
let current_byte_offset = byte_offset(rdr, rdr.pos).to_uint();
if current_byte_offset < (*rdr.src).len() {
assert!(rdr.curr != -1 as char);
let last_char = rdr.curr;
@ -202,7 +200,7 @@ pub fn is_eof(rdr: @mut StringReader) -> bool {
rdr.curr == -1 as char
}
pub fn nextch(rdr: @mut StringReader) -> char {
let offset = byte_offset(rdr).to_uint();
let offset = byte_offset(rdr, rdr.pos).to_uint();
if offset < (*rdr.src).len() {
return str::char_at(*rdr.src, offset);
} else { return -1 as char; }
@ -692,7 +690,7 @@ fn next_token_inner(rdr: @mut StringReader) -> token::Token {
return token::LIT_INT(c2 as i64, ast::ty_char);
}
'"' => {
let n = byte_offset(rdr);
let n = byte_offset(rdr, rdr.last_pos);
bump(rdr);
while rdr.curr != '"' {
if is_eof(rdr) {